From 550d4cee65757865ec483d239073983a44becf0b Mon Sep 17 00:00:00 2001 From: bplotka Date: Tue, 16 Feb 2016 18:13:22 +0100 Subject: [PATCH 1/7] WIP: Refactored SerenityConfig. Example in Overload.hpp & config_test.cpp Signed-off-by: bplotka --- src/contention_detectors/overload.cpp | 59 +++++++----- src/contention_detectors/overload.hpp | 39 ++++---- .../signal_analyzers/drop.hpp | 30 ++++--- src/filters/too_low_usage.hpp | 5 +- src/observers/strategies/cpu_contention.hpp | 4 +- src/observers/strategies/seniority.hpp | 2 +- src/pipeline/qos_pipeline.hpp | 14 +-- src/serenity/config.hpp | 89 ++++++++----------- src/serenity/resource_helper.cpp | 27 ++---- src/serenity/resource_helper.hpp | 16 ++-- src/tests/serenity/config_test.cpp | 79 ++++++++-------- 11 files changed, 172 insertions(+), 192 deletions(-) diff --git a/src/contention_detectors/overload.cpp b/src/contention_detectors/overload.cpp index 71648a2..8696157 100644 --- a/src/contention_detectors/overload.cpp +++ b/src/contention_detectors/overload.cpp @@ -5,39 +5,36 @@ #include "mesos/resources.hpp" +#include "serenity/resource_helper.hpp" + namespace mesos { namespace serenity { -Try OverloadDetector::consume(const ResourceUsage& in) { +void OverloadDetector::allProductsReady() { Contentions product; + ResourceUsage usage = getConsumable().get(); - if (in.total_size() == 0) { - return Error(std::string(NAME) + " No total in ResourceUsage"); + if (usage.total_size() == 0) { + SERENITY_LOG(ERROR) << std::string(NAME) << " No total in ResourceUsage"; + produce(product); } - Resources totalAgentResources(in.total()); + Resources totalAgentResources(usage.total()); Option totalAgentCpus = totalAgentResources.cpus(); if (totalAgentCpus.isNone()) { - return Error(std::string(NAME) + " No total cpus in ResourceUsage"); + SERENITY_LOG(ERROR) << std::string(NAME) + << " No total cpus in ResourceUsage"; + produce(product); } double_t thresholdCpus = this->cfgUtilizationThreshold * totalAgentCpus.get(); double_t agentSumCpus = 0; uint64_t beExecutors = 0; - for (const ResourceUsage_Executor& inExec : in.executors()) { - if (!inExec.has_executor_info()) { - SERENITY_LOG(ERROR) << "Executor " - << " does not include executor_info"; - // Filter out these executors. - continue; - } - if (!inExec.has_statistics()) { - SERENITY_LOG(ERROR) << "Executor " - << inExec.executor_info().executor_id().value() - << " does not include statistics."; - // Filter out these executors. + for (const ResourceUsage_Executor& inExec : usage.executors()) { + // Validate for statistics and executor info. + if (!validate(inExec)) { continue; } @@ -49,13 +46,14 @@ Try OverloadDetector::consume(const ResourceUsage& in) { agentSumCpus += value.get(); - if (!Resources(inExec.allocated()).revocable().empty()) { + if (ResourceUsageHelper::isRevocableExecutor(inExec)) { beExecutors++; } } SERENITY_LOG(INFO) << "Sum = " << agentSumCpus << " vs total = " - << totalAgentCpus.get() << " [threshold = " << thresholdCpus << "]"; + << totalAgentCpus.get() << " [threshold = " + << thresholdCpus << "]"; if (agentSumCpus > thresholdCpus) { if (beExecutors == 0) { @@ -70,12 +68,27 @@ Try OverloadDetector::consume(const ResourceUsage& in) { } } - // Continue pipeline. - this->produce(product); - - return Nothing(); + produce(product); } +bool OverloadDetector::validate(const ResourceUsage_Executor& inExec) { + if (!inExec.has_executor_info()) { + SERENITY_LOG(ERROR) << "Executor " + << " does not include executor_info"; + // Filter out these executors. + return false; + } + + if (!inExec.has_statistics()) { + SERENITY_LOG(ERROR) << "Executor " + << inExec.executor_info().executor_id().value() + << " does not include statistics."; + // Filter out these executors. + return false; + } + + return true; +} } // namespace serenity } // namespace mesos diff --git a/src/contention_detectors/overload.hpp b/src/contention_detectors/overload.hpp index 305a7d1..cca0c20 100644 --- a/src/contention_detectors/overload.hpp +++ b/src/contention_detectors/overload.hpp @@ -20,22 +20,6 @@ namespace mesos { namespace serenity { -class OverloadDetectorConfig : public SerenityConfig { - public: - OverloadDetectorConfig() { } - - explicit OverloadDetectorConfig(const SerenityConfig& customCfg) { - this->initDefaults(); - this->applyConfig(customCfg); - } - - void initDefaults() { - //! double_t - //! Detector threshold. - this->fields[detector::THRESHOLD] = - detector::DEFAULT_UTILIZATION_THRESHOLD; - } -}; /** * OverloadDetector is able to create contention if utilization is above @@ -48,23 +32,34 @@ class OverloadDetector : OverloadDetector( Consumer* _consumer, const lambda::function& _cpuUsageGetFunction, - SerenityConfig _conf, + const SerenityConfig& _conf, const Tag& _tag = Tag(QOS_CONTROLLER, NAME)) : tag(_tag), cpuUsageGetFunction(_cpuUsageGetFunction), Producer(_consumer) { - SerenityConfig config = OverloadDetectorConfig(_conf); - this->cfgUtilizationThreshold = - config.getD(detector::THRESHOLD); + configure(_conf); } ~OverloadDetector() {} - Try consume(const ResourceUsage& in) override; - static const constexpr char* NAME = "OverloadDetector"; protected: + void configure(const SerenityConfig& externalConf) { + SerenityConfig config = SerenityConfig(); + + //! double_t + //! Detector threshold. + config.set(detector::THRESHOLD, detector::DEFAULT_UTILIZATION_THRESHOLD); + + config.applyConfig(externalConf); + + // cfgUtilizationThreshold = config.getD(detector::THRESHOLD).get(); + } + + void allProductsReady() override; + bool validate(const ResourceUsage_Executor& inExec); + const Tag tag; const lambda::function cpuUsageGetFunction; diff --git a/src/contention_detectors/signal_analyzers/drop.hpp b/src/contention_detectors/signal_analyzers/drop.hpp index c423572..39688c1 100644 --- a/src/contention_detectors/signal_analyzers/drop.hpp +++ b/src/contention_detectors/signal_analyzers/drop.hpp @@ -40,28 +40,28 @@ class SignalDropAnalyzerConfig : public SerenityConfig { } void initDefaults() { - this->fields[detector::ANALYZER_TYPE] = SIGNAL_DROP_ANALYZER_NAME; + this->items[detector::ANALYZER_TYPE] = SIGNAL_DROP_ANALYZER_NAME; //! uint64_t //! How far in the past we look. - this->fields[detector::WINDOW_SIZE] = + this->items[detector::WINDOW_SIZE] = detector::DEFAULT_WINDOW_SIZE; //! double_t //! Defines how much (relatively to base point) value must drop to trigger //! contention. //! Most signal_analyzer will use that. - this->fields[detector::FRACTIONAL_THRESHOLD] = + this->items[detector::FRACTIONAL_THRESHOLD] = detector::DEFAULT_FRACTIONAL_THRESHOLD; //! double_t //! You can adjust how big severity is created for a defined drop. //! if -1 then unknown severity will be reported. - this->fields[detector::SEVERITY_FRACTION] = (double_t) -1; + this->items[detector::SEVERITY_FRACTION] = (double_t) -1; //! double_t //! Tolerance fraction of threshold if signal is accepted as returned to //! previous state after drop. - this->fields[detector::NEAR_FRACTION] = + this->items[detector::NEAR_FRACTION] = detector::DEFAULT_NEAR_FRACTION; //! uint64_t @@ -69,12 +69,12 @@ class SignalDropAnalyzerConfig : public SerenityConfig { //! Checkpoints are the reference assurance_test(base) points which we refer //! to in the past when detecting drop or not. //! It needs to be 0 < < WINDOW_SIZE - this->fields[detector::MAX_CHECKPOINTS] = + this->items[detector::MAX_CHECKPOINTS] = detector::DEFAULT_MAX_CHECKPOINTS; //! double_t //! Fraction of checkpoints' votes that important decision needs to obtain. - this->fields[detector::QUORUM] = + this->items[detector::QUORUM] = detector::DEFAULT_QUORUM; } }; @@ -110,12 +110,16 @@ class SignalDropAnalyzer : public SignalAnalyzer { valueBeforeDrop(None()), quorumNum(0) { SerenityConfig config = SignalDropAnalyzerConfig(_config); - this->cfgWindowSize = config.getU64(detector::WINDOW_SIZE); - this->cfgMaxCheckpoints = config.getU64(detector::MAX_CHECKPOINTS); - this->cfgQuroum = config.getD(detector::QUORUM); - this->cfgFractionalThreshold = config.getD(detector::FRACTIONAL_THRESHOLD); - this->cfgNearFraction = config.getD(detector::NEAR_FRACTION); - this->cfgSeverityFraction = config.getD(detector::SEVERITY_FRACTION); + this->cfgWindowSize = config.item(detector::WINDOW_SIZE).get(); + this->cfgMaxCheckpoints = + config.item(detector::MAX_CHECKPOINTS).get(); + this->cfgQuroum = config.item(detector::QUORUM).get(); + this->cfgFractionalThreshold = + config.item(detector::FRACTIONAL_THRESHOLD).get(); + this->cfgNearFraction = + config.item(detector::NEAR_FRACTION).get(); + this->cfgSeverityFraction = + config.item(detector::SEVERITY_FRACTION).get(); this->recalculateParams(); } diff --git a/src/filters/too_low_usage.hpp b/src/filters/too_low_usage.hpp index 63a5fdb..ec826d2 100644 --- a/src/filters/too_low_usage.hpp +++ b/src/filters/too_low_usage.hpp @@ -26,7 +26,7 @@ class TooLowUsageFilterConfig : public SerenityConfig { void initDefaults() { //! double_t //! Minimal cpu usage - this->fields[too_low_usage::MINIMAL_CPU_USAGE] = + this->items[too_low_usage::MINIMAL_CPU_USAGE] = too_low_usage::DEFAULT_MINIMAL_CPU_USAGE; } }; @@ -48,7 +48,8 @@ class TooLowUsageFilter : const Tag& _tag = Tag(QOS_CONTROLLER, NAME)) : Producer(_consumer), tag(_tag) { SerenityConfig config = TooLowUsageFilterConfig(_conf); - this->cfgMinimalCpuUsage = config.getD(too_low_usage::MINIMAL_CPU_USAGE); + this->cfgMinimalCpuUsage = + config.item(too_low_usage::MINIMAL_CPU_USAGE).get(); } ~TooLowUsageFilter(); diff --git a/src/observers/strategies/cpu_contention.hpp b/src/observers/strategies/cpu_contention.hpp index de3c840..4cb8867 100644 --- a/src/observers/strategies/cpu_contention.hpp +++ b/src/observers/strategies/cpu_contention.hpp @@ -27,10 +27,10 @@ class CpuContentionStrategyConfig : public SerenityConfig { // uint64_t // Specify the initial value of iterations we should wait until // we create new correction. - this->fields[strategy::CONTENTION_COOLDOWN] = + this->items[strategy::CONTENTION_COOLDOWN] = strategy::DEFAULT_CONTENTION_COOLDOWN; // double_t - this->fields[strategy::DEFAULT_CPU_SEVERITY] = + this->items[strategy::DEFAULT_CPU_SEVERITY] = strategy::DEFAULT_DEFAULT_CPU_SEVERITY; } }; diff --git a/src/observers/strategies/seniority.hpp b/src/observers/strategies/seniority.hpp index 4e72b70..7a0518c 100644 --- a/src/observers/strategies/seniority.hpp +++ b/src/observers/strategies/seniority.hpp @@ -32,7 +32,7 @@ class SeniorityStrategy : public RevocationStrategy { : RevocationStrategy(Tag(QOS_CONTROLLER, NAME)) { initialize(); if (_config.hasKey(STARTING_SEVERITY_KEY)) { - severity = _config.getD(STARTING_SEVERITY_KEY); + severity = _config.item(STARTING_SEVERITY_KEY).get(); } } diff --git a/src/pipeline/qos_pipeline.hpp b/src/pipeline/qos_pipeline.hpp index 9e16812..1fa380a 100644 --- a/src/pipeline/qos_pipeline.hpp +++ b/src/pipeline/qos_pipeline.hpp @@ -48,9 +48,9 @@ class QoSPipelineConfig : public SerenityConfig { // Used sections: QoSCorrectionObserver, AssuranceDetector, // UtilizationDetector // TODO(bplotka): Move EMA conf to separate section. - this->fields[ema::ALPHA] = ema::DEFAULT_ALPHA; - this->fields[VALVE_OPENED] = DEFAULT_VALVE_OPENED; - this->fields[ENABLED_VISUALISATION] = DEFAULT_ENABLED_VISUALISATION; + this->items[ema::ALPHA] = ema::DEFAULT_ALPHA; + this->items[VALVE_OPENED] = DEFAULT_VALVE_OPENED; + this->items[ENABLED_VISUALISATION] = DEFAULT_ENABLED_VISUALISATION; } }; @@ -138,7 +138,7 @@ class CpuQoSPipeline : public QoSControllerPipeline { &ipcDropDetector, usage::getIpc, usage::setEmaIpc, - conf.getD(ema::ALPHA_IPC), + conf.item(ema::ALPHA_IPC).get(), Tag(QOS_CONTROLLER, "ipcEMAFilter")), tooLowUsageFilter( &ipcEMAFilter, @@ -161,7 +161,7 @@ class CpuQoSPipeline : public QoSControllerPipeline { &overloadDetector, usage::getCpuUsage, usage::setEmaCpuUsage, - conf.getD(ema::ALPHA_CPU), + conf.item(ema::ALPHA_CPU).get(), Tag(QOS_CONTROLLER, "cpuEMAFilter")), cumulativeFilter( &tooLowUsageFilter, @@ -169,7 +169,7 @@ class CpuQoSPipeline : public QoSControllerPipeline { // First item in pipeline. For now, close the pipeline for QoS. valveFilter( &cumulativeFilter, - conf.getB(VALVE_OPENED), + conf.item(VALVE_OPENED).get(), Tag(QOS_CONTROLLER, "valveFilter")) { this->ageFilter.addConsumer(&valveFilter); // Setup starting producer. @@ -185,7 +185,7 @@ class CpuQoSPipeline : public QoSControllerPipeline { cumulativeFilter.addConsumer(&cpuEMAFilter); // Setup Time Series export - if (conf.getB(ENABLED_VISUALISATION)) { + if (conf.item(ENABLED_VISUALISATION).get()) { this->addConsumer(&rawResourcesExporter); ipcEMAFilter.addConsumer(&emaFilteredResourcesExporter); } diff --git a/src/serenity/config.hpp b/src/serenity/config.hpp index 3b40105..f302574 100644 --- a/src/serenity/config.hpp +++ b/src/serenity/config.hpp @@ -11,25 +11,24 @@ #include "serenity/serenity.hpp" #include "stout/option.hpp" +#include "stout/result.hpp" namespace mesos { namespace serenity { /** * Global Serenity Config class which implements basic mechanism - * to support specifying config parameters via string key map. + * to support specifying config parameters via string key map & sections. * * Check config_test.cpp to see example usage. - * - * TODO(skonefal): every getter should pack result in Try. */ class SerenityConfig { public: SerenityConfig() {} /** - * Variant type for storing multiple types of data in configuration. - */ + * Variant type for storing multiple types of data in configuration. + */ using CfgVariant = boost::variant< bool, int64_t, uint64_t, double_t, std::string>; @@ -37,14 +36,32 @@ class SerenityConfig { * Overlapping custom configuration options using recursive copy. */ void applyConfig(const SerenityConfig& customCfg) { - this->recursiveCfgCopy(this, customCfg); + recursiveCfgCopy(this, customCfg); } /** - * Gets variant config value. + * Templated, safe getter for item in config. */ - Option operator()(std::string key) const { - return getField(key); + template + const Result item(std::string key) const { + Result result = None(); + + // Get item from items map. + Option variantResult = getItem(key); + + if (variantResult.isSome()) { + // When item is found, try to parse it to the specified T type. + try { + result = boost::get(variantResult.get()); + } catch (std::exception& e) { + // NOTE(bplotka): Log here???? + LOG(ERROR) << "Failed to parse " << key + << " field: " << e.what(); + result = Result::error(e.what()); + } + } + + return result; } /** @@ -55,42 +72,6 @@ class SerenityConfig { return *getSection(key); } - // -- unsafe getters -- - - /** - * Unsafe getter for string - */ - std::string getS(std::string key) { - return boost::get(this->fields[key]); - } - - /** - * Unsafe getter for int64_t - */ - int64_t getI64(std::string key) { - return boost::get(this->fields[key]); - } - - /** - * Unsafe getter for uint64_t - */ - uint64_t getU64(std::string key) { - return boost::get(this->fields[key]); - } - - /** - * Unsafe getter for double_t - */ - double_t getD(std::string key) { - return boost::get(this->fields[key]); - } - - /** - * Unsafe getter for bool - */ - bool getB(std::string key) { - return boost::get(this->fields[key]); - } // -- setters -- @@ -140,11 +121,11 @@ class SerenityConfig { * Sets CfgVariant config value. */ void setVariant(std::string key, SerenityConfig::CfgVariant value) { - this->fields[key] = value; + this->items[key] = value; } - bool hasKey(std::string key) { - return fields.find(key) != fields.end(); + bool hasKey(std::string key) const { + return items.find(key) != items.end(); } /** @@ -159,7 +140,7 @@ class SerenityConfig { }; protected: - std::unordered_map fields; + std::unordered_map items; /** * Support for hierarchical configuration sections. @@ -186,9 +167,9 @@ class SerenityConfig { /** * Getter for field. */ - Option getField(std::string fieldKey) const { - auto mapItem = this->fields.find(fieldKey); - if (mapItem != this->fields.end()) { + Option getItem(std::string itemKey) const { + auto mapItem = this->items.find(itemKey); + if (mapItem != this->items.end()) { return mapItem->second; } @@ -201,8 +182,8 @@ class SerenityConfig { */ void recursiveCfgCopy(SerenityConfig* base, const SerenityConfig& customCfg) const { - for (auto customItem : customCfg.fields) { - base->fields[customItem.first] = customItem.second; + for (auto customItem : customCfg.items) { + base->items[customItem.first] = customItem.second; } for (auto customSection : customCfg.sections) { diff --git a/src/serenity/resource_helper.cpp b/src/serenity/resource_helper.cpp index b105c6f..5d727b4 100644 --- a/src/serenity/resource_helper.cpp +++ b/src/serenity/resource_helper.cpp @@ -43,36 +43,19 @@ ResourceUsageHelper::getProductionAndRevocableExecutors( revocableExecutors); } -Try ResourceUsageHelper::isProductionExecutor( +bool ResourceUsageHelper::isProductionExecutor( const ResourceUsage_Executor& executor) { - if (executor.allocated().size() == 0) { - return Error("Executor has no allocated resources."); - } - - if (Resources(executor.allocated()).revocable().empty()) { - return true; - } else { - return false; - } + return Resources(executor.allocated()).revocable().empty(); } bool ResourceUsageHelper::isExecutorHasStatistics( const ResourceUsage_Executor& executor) { - if (executor.has_executor_info() && executor.has_statistics()) { - return true; - } else { - return false; - } + return executor.has_executor_info() && executor.has_statistics(); } -Try ResourceUsageHelper::isRevocableExecutor( +bool ResourceUsageHelper::isRevocableExecutor( const ResourceUsage_Executor &executor) { - Try result = isProductionExecutor(executor); - if (result.isError()) { - return result; - } - - return !result.get(); + return !isProductionExecutor(executor); } } // namespace serenity diff --git a/src/serenity/resource_helper.hpp b/src/serenity/resource_helper.hpp index bff7a9d..05823ae 100644 --- a/src/serenity/resource_helper.hpp +++ b/src/serenity/resource_helper.hpp @@ -34,18 +34,14 @@ class ResourceUsageHelper { getProductionAndRevocableExecutors(const ResourceUsage&); /** - * Checks if executor has empty revocable resources. - * - * Returns error when executor has no allocated resources. - */ - static Try isProductionExecutor(const ResourceUsage_Executor&); + * Checks if executor has empty revocable resources. + */ + static bool isProductionExecutor(const ResourceUsage_Executor&); /** - * Checks if executor has revocable resources. - * - * Returns error when executor has no allocated resources. - */ - static Try isRevocableExecutor(const ResourceUsage_Executor&); + * Checks if executor has revocable resources. + */ + static bool isRevocableExecutor(const ResourceUsage_Executor&); static bool isExecutorHasStatistics(const ResourceUsage_Executor&); diff --git a/src/tests/serenity/config_test.cpp b/src/tests/serenity/config_test.cpp index 923579a..94b4d18 100644 --- a/src/tests/serenity/config_test.cpp +++ b/src/tests/serenity/config_test.cpp @@ -11,7 +11,7 @@ namespace mesos { namespace serenity { namespace tests { -// TestConfig required fields & default values using different types. +// TestConfig required items & default values using different types. const constexpr char* FIELD_STR = "FIELD_STR"; const constexpr char* DEFAULT_FIELD_STR = "default"; const constexpr char* MODIFIED_FIELD_STR = "modified"; @@ -33,49 +33,51 @@ const constexpr double_t DEFAULT_FIELD_DOUBLE = 0.345345; const constexpr double_t MODIFIED_FIELD_DOUBLE = 3.432; -class TestConfig : public SerenityConfig { +class TestConfigFilter { public: - TestConfig() { - this->initDefaults(); + explicit TestConfigFilter(const SerenityConfig& customCfg) { + configure(customCfg); } - /** - * This constructor enables run-time overlapping of default - * configuration records. - */ - explicit TestConfig(const SerenityConfig& customCfg) { - this->initDefaults(); - this->applyConfig(customCfg); - } + void configure(const SerenityConfig& externalConf) { + config = SerenityConfig(); + + config.set(FIELD_STR, (std::string)DEFAULT_FIELD_STR); + config.set(FIELD_BOOL, DEFAULT_FIELD_BOOL); + config.set(FIELD_UINT, DEFAULT_FIELD_UINT); + config.set(FIELD_INT, DEFAULT_FIELD_INT); + config.set(FIELD_DOUBLE, DEFAULT_FIELD_DOUBLE); - /** - * Init default values for Test configuration. - */ - void initDefaults() { - this->set(FIELD_STR, (std::string)DEFAULT_FIELD_STR); - this->set(FIELD_BOOL, DEFAULT_FIELD_BOOL); - this->set(FIELD_UINT, DEFAULT_FIELD_UINT); - this->set(FIELD_INT, DEFAULT_FIELD_INT); - this->set(FIELD_DOUBLE, DEFAULT_FIELD_DOUBLE); + config.applyConfig(externalConf); } + + SerenityConfig config; }; TEST(SerenityConfigTest, DefaultValuesAvailable) { - // Create empty config with no configuration fields. + // Create empty config with no configuration items. SerenityConfig newConfig; - TestConfig internalConfig = TestConfig(newConfig); - EXPECT_EQ(internalConfig.getS(FIELD_STR), (std::string)DEFAULT_FIELD_STR); - EXPECT_EQ(internalConfig.getB(FIELD_BOOL), DEFAULT_FIELD_BOOL); - EXPECT_EQ(internalConfig.getU64(FIELD_UINT), DEFAULT_FIELD_UINT); - EXPECT_EQ(internalConfig.getI64(FIELD_INT), DEFAULT_FIELD_INT); - EXPECT_EQ(internalConfig.getD(FIELD_DOUBLE), DEFAULT_FIELD_DOUBLE); + TestConfigFilter testFilter = TestConfigFilter(newConfig); + EXPECT_EQ(testFilter.config.item(FIELD_STR).get(), + (std::string) DEFAULT_FIELD_STR); + EXPECT_EQ(testFilter.config.item(FIELD_BOOL).get(), + DEFAULT_FIELD_BOOL); + EXPECT_EQ(testFilter.config.item(FIELD_UINT).get(), + DEFAULT_FIELD_UINT); + EXPECT_EQ(testFilter.config.item(FIELD_INT).get(), + DEFAULT_FIELD_INT); + EXPECT_EQ(testFilter.config.item(FIELD_DOUBLE).get(), + DEFAULT_FIELD_DOUBLE); + + EXPECT_EQ(testFilter.config.item(FIELD_BOOL).get(), DEFAULT_FIELD_BOOL); + } TEST(SerenityConfigTest, ModifiedValuesAvailable) { - // Create config with custom configuration fields. + // Create config with custom configuration items. SerenityConfig newConfig; newConfig.set(FIELD_STR, (std::string)MODIFIED_FIELD_STR); newConfig.set(FIELD_BOOL, MODIFIED_FIELD_BOOL); @@ -83,15 +85,20 @@ TEST(SerenityConfigTest, ModifiedValuesAvailable) { newConfig.set(FIELD_INT, MODIFIED_FIELD_INT); newConfig.set(FIELD_DOUBLE, MODIFIED_FIELD_DOUBLE); - SerenityConfig internalConfig = TestConfig(newConfig); - EXPECT_EQ(internalConfig.getS(FIELD_STR), (std::string)MODIFIED_FIELD_STR); - EXPECT_EQ(internalConfig.getB(FIELD_BOOL), MODIFIED_FIELD_BOOL); - EXPECT_EQ(internalConfig.getU64(FIELD_UINT), MODIFIED_FIELD_UINT); - EXPECT_EQ(internalConfig.getI64(FIELD_INT), MODIFIED_FIELD_INT); - EXPECT_EQ(internalConfig.getD(FIELD_DOUBLE), MODIFIED_FIELD_DOUBLE); + TestConfigFilter testFilter = TestConfigFilter(newConfig); + + EXPECT_EQ(testFilter.config.item(FIELD_STR).get(), + (std::string) MODIFIED_FIELD_STR); + EXPECT_EQ(testFilter.config.item(FIELD_BOOL).get(), + MODIFIED_FIELD_BOOL); + EXPECT_EQ(testFilter.config.item(FIELD_UINT).get(), + MODIFIED_FIELD_UINT); + EXPECT_EQ(testFilter.config.item(FIELD_INT).get(), + MODIFIED_FIELD_INT); + EXPECT_EQ(testFilter.config.item(FIELD_DOUBLE).get(), + MODIFIED_FIELD_DOUBLE); } } // namespace tests } // namespace serenity } // namespace mesos - From f19fd5f1d1a51f9422fb5a4427223ad4d36542c0 Mon Sep 17 00:00:00 2001 From: bplotka Date: Tue, 16 Feb 2016 18:14:07 +0100 Subject: [PATCH 2/7] Fixed lint issue. Signed-off-by: bplotka --- src/tests/serenity/config_test.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/tests/serenity/config_test.cpp b/src/tests/serenity/config_test.cpp index 94b4d18..12dc367 100644 --- a/src/tests/serenity/config_test.cpp +++ b/src/tests/serenity/config_test.cpp @@ -72,7 +72,6 @@ TEST(SerenityConfigTest, DefaultValuesAvailable) { DEFAULT_FIELD_DOUBLE); EXPECT_EQ(testFilter.config.item(FIELD_BOOL).get(), DEFAULT_FIELD_BOOL); - } From fde11f8c34842c8631bce62fd03dc2299eb7c7c6 Mon Sep 17 00:00:00 2001 From: bplotka Date: Wed, 17 Feb 2016 20:40:18 +0100 Subject: [PATCH 3/7] Refactored SerenityConfing, templated set & item Signed-off-by: bplotka --- src/contention_detectors/overload.cpp | 2 +- src/contention_detectors/overload.hpp | 19 +-- .../signal_analyzers/drop.hpp | 136 +++++++++--------- .../serenity_controller_module.cpp | 42 +----- src/pipeline/qos_pipeline.hpp | 41 ++---- src/serenity/config.hpp | 97 +++++++------ src/serenity/default_vars.hpp | 22 ++- src/serenity/serenity.hpp | 17 +++ src/tests/common/config_helper.hpp | 14 +- src/tests/serenity/config_test.cpp | 97 ++++++------- 10 files changed, 237 insertions(+), 250 deletions(-) diff --git a/src/contention_detectors/overload.cpp b/src/contention_detectors/overload.cpp index 8696157..2a641a4 100644 --- a/src/contention_detectors/overload.cpp +++ b/src/contention_detectors/overload.cpp @@ -16,7 +16,7 @@ void OverloadDetector::allProductsReady() { if (usage.total_size() == 0) { SERENITY_LOG(ERROR) << std::string(NAME) << " No total in ResourceUsage"; - produce(product); + produce(Contentions()); } Resources totalAgentResources(usage.total()); diff --git a/src/contention_detectors/overload.hpp b/src/contention_detectors/overload.hpp index cca0c20..077dbc5 100644 --- a/src/contention_detectors/overload.hpp +++ b/src/contention_detectors/overload.hpp @@ -37,26 +37,21 @@ class OverloadDetector : : tag(_tag), cpuUsageGetFunction(_cpuUsageGetFunction), Producer(_consumer) { - configure(_conf); + // Parse config values. + setCfgUtilizationThreshold( + _conf.item(detector::THRESHOLD, + detector::DEFAULT_UTILIZATION_THRESHOLD)); } ~OverloadDetector() {} static const constexpr char* NAME = "OverloadDetector"; - protected: - void configure(const SerenityConfig& externalConf) { - SerenityConfig config = SerenityConfig(); - - //! double_t - //! Detector threshold. - config.set(detector::THRESHOLD, detector::DEFAULT_UTILIZATION_THRESHOLD); - - config.applyConfig(externalConf); - - // cfgUtilizationThreshold = config.getD(detector::THRESHOLD).get(); + void setCfgUtilizationThreshold(double_t cfgUtilizationThreshold) { + OverloadDetector::cfgUtilizationThreshold = cfgUtilizationThreshold; } + protected: void allProductsReady() override; bool validate(const ResourceUsage_Executor& inExec); diff --git a/src/contention_detectors/signal_analyzers/drop.hpp b/src/contention_detectors/signal_analyzers/drop.hpp index 39688c1..82ea391 100644 --- a/src/contention_detectors/signal_analyzers/drop.hpp +++ b/src/contention_detectors/signal_analyzers/drop.hpp @@ -30,56 +30,6 @@ namespace serenity { #define SIGNAL_DROP_ANALYZER_NAME "AssuranceDropAnalyzer" -class SignalDropAnalyzerConfig : public SerenityConfig { - public: - SignalDropAnalyzerConfig() {} - - explicit SignalDropAnalyzerConfig(const SerenityConfig& customCfg) { - this->initDefaults(); - this->applyConfig(customCfg); - } - - void initDefaults() { - this->items[detector::ANALYZER_TYPE] = SIGNAL_DROP_ANALYZER_NAME; - //! uint64_t - //! How far in the past we look. - this->items[detector::WINDOW_SIZE] = - detector::DEFAULT_WINDOW_SIZE; - - //! double_t - //! Defines how much (relatively to base point) value must drop to trigger - //! contention. - //! Most signal_analyzer will use that. - this->items[detector::FRACTIONAL_THRESHOLD] = - detector::DEFAULT_FRACTIONAL_THRESHOLD; - - //! double_t - //! You can adjust how big severity is created for a defined drop. - //! if -1 then unknown severity will be reported. - this->items[detector::SEVERITY_FRACTION] = (double_t) -1; - - //! double_t - //! Tolerance fraction of threshold if signal is accepted as returned to - //! previous state after drop. - this->items[detector::NEAR_FRACTION] = - detector::DEFAULT_NEAR_FRACTION; - - //! uint64_t - //! Maximum number of checkpoints we will have in our assurance detector. - //! Checkpoints are the reference assurance_test(base) points which we refer - //! to in the past when detecting drop or not. - //! It needs to be 0 < < WINDOW_SIZE - this->items[detector::MAX_CHECKPOINTS] = - detector::DEFAULT_MAX_CHECKPOINTS; - - //! double_t - //! Fraction of checkpoints' votes that important decision needs to obtain. - this->items[detector::QUORUM] = - detector::DEFAULT_QUORUM; - } -}; - - /** * Dynamic implementation of sequential change point detection. * @@ -109,17 +59,30 @@ class SignalDropAnalyzer : public SignalAnalyzer { : SignalAnalyzer(_tag), valueBeforeDrop(None()), quorumNum(0) { - SerenityConfig config = SignalDropAnalyzerConfig(_config); - this->cfgWindowSize = config.item(detector::WINDOW_SIZE).get(); - this->cfgMaxCheckpoints = - config.item(detector::MAX_CHECKPOINTS).get(); - this->cfgQuroum = config.item(detector::QUORUM).get(); - this->cfgFractionalThreshold = - config.item(detector::FRACTIONAL_THRESHOLD).get(); - this->cfgNearFraction = - config.item(detector::NEAR_FRACTION).get(); - this->cfgSeverityFraction = - config.item(detector::SEVERITY_FRACTION).get(); + + setCfgWindowSize(_config.item( + detector::WINDOW_SIZE, + detector::DEFAULT_WINDOW_SIZE)); + + setCfgMaxCheckpoints(_config.item( + detector::MAX_CHECKPOINTS, + detector::DEFAULT_MAX_CHECKPOINTS)); + + setCfgQuroum(_config.item( + detector::QUORUM, + detector::DEFAULT_QUORUM)); + + setCfgFractionalThreshold(_config.item( + detector::FRACTIONAL_THRESHOLD, + detector::DEFAULT_FRACTIONAL_THRESHOLD)); + + setCfgNearFraction(_config.item( + detector::NEAR_FRACTION, + detector::DEFAULT_NEAR_FRACTION)); + + setCfgNearFraction(_config.item( + detector::SEVERITY_FRACTION, + detector::DEFAULT_NEAR_FRACTION)); this->recalculateParams(); } @@ -135,6 +98,49 @@ class SignalDropAnalyzer : public SignalAnalyzer { */ void shiftBasePoints(); + //! int64_t + //! How far in the past we look. + void setCfgWindowSize(int64_t cfgWindowSize) { + SignalDropAnalyzer::cfgWindowSize = cfgWindowSize; + } + + //! int64_t + //! Maximum number of checkpoints we will have in our assurance detector. + //! Checkpoints are the reference assurance_test(base) points which we refer + //! to in the past when detecting drop or not. + //! It needs to be 0 < < WINDOW_SIZE + void setCfgMaxCheckpoints(int64_t cfgMaxCheckpoints) { + SignalDropAnalyzer::cfgMaxCheckpoints = cfgMaxCheckpoints; + } + + //! double_t + //! Fraction of checkpoints' votes that important decision needs to obtain. + void setCfgQuroum(double_t cfgQuroum) { + SignalDropAnalyzer::cfgQuroum = cfgQuroum; + } + + //! double_t + //! Defines how much (relatively to base point) value must drop to trigger + //! contention. + //! Most signal_analyzer will use that. + void setCfgFractionalThreshold(double_t cfgFractionalThreshold) { + SignalDropAnalyzer::cfgFractionalThreshold = cfgFractionalThreshold; + } + + //! double_t + //! You can adjust how big severity is created for a defined drop. + //! if -1 then unknown severity will be reported. + void setCfgSeverityFraction(double_t cfgSeverityFraction) { + SignalDropAnalyzer::cfgSeverityFraction = cfgSeverityFraction; + } + + //! double_t + //! Tolerance fraction of threshold if signal is accepted as returned to + //! previous state after drop. + void setCfgNearFraction(double_t cfgNearFraction) { + SignalDropAnalyzer::cfgNearFraction = cfgNearFraction; + } + protected: std::list window; std::list::iterator> basePoints; @@ -146,16 +152,16 @@ class SignalDropAnalyzer : public SignalAnalyzer { uint32_t quorumNum; // cfg parameters. - uint64_t cfgWindowSize; - uint64_t cfgMaxCheckpoints; + int64_t cfgWindowSize; + int64_t cfgMaxCheckpoints; double_t cfgQuroum; double_t cfgFractionalThreshold; double_t cfgSeverityFraction; double_t cfgNearFraction; /** - * It is possible to dynamically change analyzer configuration. - */ + * It is possible to dynamically change analyzer configuration. + */ void recalculateParams(); }; diff --git a/src/mesos_modules/qos_controller/serenity_controller_module.cpp b/src/mesos_modules/qos_controller/serenity_controller_module.cpp index 82cc197..fef8c19 100644 --- a/src/mesos_modules/qos_controller/serenity_controller_module.cpp +++ b/src/mesos_modules/qos_controller/serenity_controller_module.cpp @@ -40,47 +40,17 @@ static QoSController* createSerenityController( const Parameters& parameters) { LOG(INFO) << "Loading Serenity QoS Controller module"; // TODO(bplotka): Fetch configuration from parameters or conf file. - // - // --Hardcoded configuration for Serenity QoS Controller--- SerenityConfig conf; - // AssuranceDropAnalyzer configuration: - // How far we look back in samples. - conf[SIGNAL_DROP_ANALYZER_NAME].set(WINDOW_SIZE, (uint64_t) 10); - // Defines how much (relatively to base point) value must drop to trigger - // contention. - // Most signal_analyzer will use that. - conf[SIGNAL_DROP_ANALYZER_NAME].set(FRACTIONAL_THRESHOLD, (double_t) 0.3); - conf[SIGNAL_DROP_ANALYZER_NAME].set(SEVERITY_FRACTION, (double_t) 2.1); - - // How many iterations observers will wait with creating another - // correction. - conf[CpuContentionStrategy::NAME].set(CONTENTION_COOLDOWN, (uint64_t) 10); - conf[SeniorityStrategy::NAME].set(CONTENTION_COOLDOWN, (uint64_t) 10); - - // UtilizationDetector configuration: - // CPU utilization threshold. - conf[THRESHOLD].set(THRESHOLD, (double_t) 0.72); - - conf[TooLowUsageFilter::NAME].set(MINIMAL_CPU_USAGE, (double_t) 0.25); - - conf.set(ALPHA_CPU, (double_t) 0.9); - conf.set(ALPHA_IPC, (double_t) 0.9); - conf.set(ENABLED_VISUALISATION, false); - conf.set(VALVE_OPENED, true); - - // Since slave is configured for 5 second perf interval, it is useless to - // check correction more often then 5 sec. - double onEmptyCorrectionInterval = 2; - - // --End of hardcoded configuration for Serenity QoS Controller--- + double onEmptyCorrectionInterval = + conf.item(ON_EMPTY_CORRECTION_INTERVAL, + DEFAULT_ON_EMPTY_CORRECTION_INTERVAL); // Use static constructor of QoSController. Try result = - SerenityController::create( - std::shared_ptr( - new CpuQoSPipeline(conf)), - onEmptyCorrectionInterval); + SerenityController::create(std::shared_ptr( + new CpuQoSPipeline(conf)), + onEmptyCorrectionInterval); if (result.isError()) { return NULL; diff --git a/src/pipeline/qos_pipeline.hpp b/src/pipeline/qos_pipeline.hpp index 1fa380a..6b0bd46 100644 --- a/src/pipeline/qos_pipeline.hpp +++ b/src/pipeline/qos_pipeline.hpp @@ -28,8 +28,6 @@ #include "serenity/data_utils.hpp" #include "serenity/serenity.hpp" -#include "time_series_export/resource_usage_ts_export.hpp" - namespace mesos { namespace serenity { @@ -105,10 +103,7 @@ using QoSControllerPipeline = Pipeline; class CpuQoSPipeline : public QoSControllerPipeline { public: explicit CpuQoSPipeline(const SerenityConfig& _conf) - : conf(QoSPipelineConfig(_conf)), - // Time series exporters. - rawResourcesExporter("raw"), - emaFilteredResourcesExporter("ema"), + : conf(_conf), // NOTE(bplotka): age Filter should initialized first before passing // to the qosCorrectionObserver. ageFilter(), @@ -116,12 +111,12 @@ class CpuQoSPipeline : public QoSControllerPipeline { correctionMerger( this, Tag(QOS_CONTROLLER, "CorrectionMerger")), -// ipcContentionObserver( -// &correctionMerger, -// &ageFilter, -// new SeniorityStrategy(conf[SeniorityStrategy::NAME]), -// strategy::DEFAULT_CONTENTION_COOLDOWN, -// Tag(QOS_CONTROLLER, SeniorityStrategy::NAME)), + // ipcContentionObserver( + // &correctionMerger, + // &ageFilter, + // new SeniorityStrategy(conf[SeniorityStrategy::NAME]), + // strategy::DEFAULT_CONTENTION_COOLDOWN, + // Tag(QOS_CONTROLLER, SeniorityStrategy::NAME)), cacheOccupancyContentionObserver( &correctionMerger, &ageFilter, @@ -138,7 +133,7 @@ class CpuQoSPipeline : public QoSControllerPipeline { &ipcDropDetector, usage::getIpc, usage::setEmaIpc, - conf.item(ema::ALPHA_IPC).get(), + conf.item(ema::ALPHA_IPC, ema::DEFAULT_ALPHA_IPC), Tag(QOS_CONTROLLER, "ipcEMAFilter")), tooLowUsageFilter( &ipcEMAFilter, @@ -161,7 +156,7 @@ class CpuQoSPipeline : public QoSControllerPipeline { &overloadDetector, usage::getCpuUsage, usage::setEmaCpuUsage, - conf.item(ema::ALPHA_CPU).get(), + conf.item(ema::ALPHA_CPU, ema::DEFAULT_ALPHA_CPU), Tag(QOS_CONTROLLER, "cpuEMAFilter")), cumulativeFilter( &tooLowUsageFilter, @@ -169,35 +164,25 @@ class CpuQoSPipeline : public QoSControllerPipeline { // First item in pipeline. For now, close the pipeline for QoS. valveFilter( &cumulativeFilter, - conf.item(VALVE_OPENED).get(), + conf.item(VALVE_OPENED, DEFAULT_VALVE_OPENED), Tag(QOS_CONTROLLER, "valveFilter")) { this->ageFilter.addConsumer(&valveFilter); // Setup starting producer. this->addConsumer(&ageFilter); -// cacheOccupancyContentionObserver. -// Producer::addConsumer(&ipcContentionObserver); + // cacheOccupancyContentionObserver. + // Producer::addConsumer(&ipcContentionObserver); // QoSCorrection observers needs ResourceUsage as well. cpuEMAFilter.addConsumer(&cpuContentionObserver); -// cumulativeFilter.addConsumer(&ipcContentionObserver); + // cumulativeFilter.addConsumer(&ipcContentionObserver); cumulativeFilter.addConsumer(&cacheOccupancyContentionObserver); cumulativeFilter.addConsumer(&cpuEMAFilter); - - // Setup Time Series export - if (conf.item(ENABLED_VISUALISATION).get()) { - this->addConsumer(&rawResourcesExporter); - ipcEMAFilter.addConsumer(&emaFilteredResourcesExporter); - } } private: SerenityConfig conf; - // --- Time Series Exporters --- - ResourceUsageTimeSeriesExporter rawResourcesExporter; - ResourceUsageTimeSeriesExporter emaFilteredResourcesExporter; - // --- Shared resource contention QoS CorrectionMergerFilter correctionMerger; // QoSCorrectionObserver ipcContentionObserver; diff --git a/src/serenity/config.hpp b/src/serenity/config.hpp index f302574..d75bf4e 100644 --- a/src/serenity/config.hpp +++ b/src/serenity/config.hpp @@ -4,6 +4,7 @@ #include #include #include +#include #include "boost/variant.hpp" @@ -17,7 +18,7 @@ namespace mesos { namespace serenity { /** - * Global Serenity Config class which implements basic mechanism + * Serenity Config class which implements basic mechanism * to support specifying config parameters via string key map & sections. * * Check config_test.cpp to see example usage. @@ -30,7 +31,7 @@ class SerenityConfig { * Variant type for storing multiple types of data in configuration. */ using CfgVariant = boost::variant< - bool, int64_t, uint64_t, double_t, std::string>; + bool, int64_t, double_t, std::string>; /** * Overlapping custom configuration options using recursive copy. @@ -43,7 +44,14 @@ class SerenityConfig { * Templated, safe getter for item in config. */ template - const Result item(std::string key) const { + const Result item(const std::string& key) const { + static_assert(std::is_same() + || std::is_same() + || std::is_same() + || std::is_same(), + "T must be one of the types stored in CfgVariant (bool, " + "int64_t, double_t, string)"); + Result result = None(); // Get item from items map. @@ -54,9 +62,8 @@ class SerenityConfig { try { result = boost::get(variantResult.get()); } catch (std::exception& e) { - // NOTE(bplotka): Log here???? LOG(ERROR) << "Failed to parse " << key - << " field: " << e.what(); + << " field: " << e.what(); result = Result::error(e.what()); } } @@ -65,66 +72,72 @@ class SerenityConfig { } /** - * Gets config section. - * In case there is not one, create empty section. + * Templated, safe getter for item in config. Sets default value in case of + * error or none. */ - SerenityConfig& operator[](std::string key) { - return *getSection(key); - } - - - // -- setters -- + template + const T item(const std::string& key, T defaultValue) const { + static_assert(std::is_same() + || std::is_same() + || std::is_same() + || std::is_same(), + "T must be one of the types stored in CfgVariant (bool, " + "int64_t, double_t, string)"); + T result = defaultValue; - /** - * Sets char* config value. - */ - void set(std::string key, char* value) { - this->setVariant(key, (std::string)value); - } + // Get item from items map. + Option variantResult = getItem(key); - /** - * Sets string config value. - */ - void set(std::string key, std::string value) { - this->setVariant(key, value); - } + if (variantResult.isSome()) { + // When item is found, try to parse it to the specified T type. + try { + result = boost::get(variantResult.get()); + } catch (std::exception& e) { + LOG(ERROR) << "Failed to parse " << key << " to type " + << typeid(T).name() << ". Field: " << e.what(); + } + } - /** - * Sets bool config value. - */ - void set(std::string key, bool value) { - this->setVariant(key, value); + return result; } /** - * Sets uint64_t config value. + * Gets config section. + * In case there is not one, create empty section. */ - void set(std::string key, uint64_t value) { - this->setVariant(key, value); + SerenityConfig& operator[](const std::string& key) { + return *getSection(key); } /** - * Sets int64_t config value. + * Templated set config value. */ - void set(std::string key, int64_t value) { + template + void set(const std::string& key, T value) { + static_assert(std::is_same() + || std::is_same() + || std::is_same() + || std::is_same(), + "T must be one of the types stored in CfgVariant (bool, " + "int64_t, double_t, string)"); this->setVariant(key, value); } /** - * Sets double_t config value. + * Templated set config value for char*. */ - void set(std::string key, double_t value) { - this->setVariant(key, value); + void set(const std::string& key, char* value) { + this->setVariant(key, (std::string) value); } /** * Sets CfgVariant config value. */ - void setVariant(std::string key, SerenityConfig::CfgVariant value) { + void setVariant(const std::string& key, SerenityConfig::CfgVariant value) { this->items[key] = value; } - bool hasKey(std::string key) const { + bool hasKey(const std::string& key) const { return items.find(key) != items.end(); } @@ -151,7 +164,7 @@ class SerenityConfig { * Getter for section. * In case of no section - create such. */ - std::shared_ptr getSection(std::string sectionKey) { + std::shared_ptr getSection(const std::string& sectionKey) { auto mapItem = this->sections.find(sectionKey); if (mapItem != this->sections.end()) { return mapItem->second; @@ -167,7 +180,7 @@ class SerenityConfig { /** * Getter for field. */ - Option getItem(std::string itemKey) const { + Option getItem(const std::string& itemKey) const { auto mapItem = this->items.find(itemKey); if (mapItem != this->items.end()) { return mapItem->second; diff --git a/src/serenity/default_vars.hpp b/src/serenity/default_vars.hpp index bfd6a47..03fb202 100644 --- a/src/serenity/default_vars.hpp +++ b/src/serenity/default_vars.hpp @@ -11,7 +11,13 @@ namespace qos_pipeline { const constexpr char* VALVE_OPENED = "VALVE_OPENED"; constexpr bool DEFAULT_VALVE_OPENED = true; const constexpr char* ENABLED_VISUALISATION = "ENABLED_VISUALISATION"; -constexpr bool DEFAULT_ENABLED_VISUALISATION = true; +constexpr bool DEFAULT_ENABLED_VISUALISATION = false; + +const constexpr char* ON_EMPTY_CORRECTION_INTERVAL = + "ON_EMPTY_CORRECTION_INTERVAL"; +constexpr double_t DEFAULT_ON_EMPTY_CORRECTION_INTERVAL = 2; + + } // namespace qos_pipeline @@ -25,29 +31,31 @@ const constexpr char* ALPHA = "ALPHA"; constexpr double_t DEFAULT_ALPHA = 0.2; const constexpr char* ALPHA_CPU = "ALPHA_CPU"; +constexpr double_t DEFAULT_ALPHA_CPU = 0.9; const constexpr char* ALPHA_IPC = "ALPHA_IPC"; +constexpr double_t DEFAULT_ALPHA_IPC = 0.9; } // namespace ema namespace detector { const constexpr char* ANALYZER_TYPE = "ANALYZER_TYPE"; const constexpr char* WINDOW_SIZE = "WINDOW_SIZE"; -constexpr uint64_t DEFAULT_WINDOW_SIZE = 10; +constexpr int64_t DEFAULT_WINDOW_SIZE = 10; const constexpr char* FRACTIONAL_THRESHOLD = "FRACTIONAL_THRESHOLD"; -constexpr double_t DEFAULT_FRACTIONAL_THRESHOLD = 0.5; +constexpr double_t DEFAULT_FRACTIONAL_THRESHOLD = 0.3; const constexpr char* SEVERITY_FRACTION = "SEVERITY_FRACTION"; -constexpr double_t DEFAULT_SEVERITY_FRACTION = -1; +constexpr double_t DEFAULT_SEVERITY_FRACTION = 2.1; const constexpr char* NEAR_FRACTION = "NEAR_FRACTION"; constexpr double_t DEFAULT_NEAR_FRACTION = 0.1; const constexpr char* MAX_CHECKPOINTS = "MAX_CHECKPOINTS"; -constexpr uint64_t DEFAULT_MAX_CHECKPOINTS = 3; +constexpr int64_t DEFAULT_MAX_CHECKPOINTS = 3; const constexpr char* QUORUM = "QUORUM"; constexpr double_t DEFAULT_QUORUM = 0.70; constexpr double_t DEFAULT_START_VALUE = 0.00001; const constexpr char* THRESHOLD = "THRESHOLD"; -constexpr double_t DEFAULT_UTILIZATION_THRESHOLD = 0.85; +constexpr double_t DEFAULT_UTILIZATION_THRESHOLD = 0.72; } // namespace detector namespace slack_observer { @@ -70,7 +78,7 @@ constexpr double_t DEFAULT_MINIMAL_CPU_USAGE = 0.25; // !< per sec. namespace strategy { const constexpr char* CONTENTION_COOLDOWN = "CONTENTION_COOLDOWN"; -constexpr uint64_t DEFAULT_CONTENTION_COOLDOWN = 10; +constexpr int64_t DEFAULT_CONTENTION_COOLDOWN = 10; const constexpr char* DEFAULT_CPU_SEVERITY = "DEFAULT_CPU_SEVERITY"; constexpr double_t DEFAULT_DEFAULT_CPU_SEVERITY = 1.0; static const constexpr char* STARTING_SEVERITY = "STARTING_SEVERITY"; diff --git a/src/serenity/serenity.hpp b/src/serenity/serenity.hpp index 01640b3..f777e7b 100644 --- a/src/serenity/serenity.hpp +++ b/src/serenity/serenity.hpp @@ -8,6 +8,7 @@ #include "stout/nothing.hpp" #include "stout/try.hpp" +#include "stout/result.hpp" namespace mesos { namespace serenity { @@ -34,6 +35,22 @@ class BaseFilter { virtual ~BaseFilter() {} + /** + * Getting Result safely with default value when in error or none state. + */ + template + T safeGetResult(Result result, T defaultValue) { + if (result.isSome()) { + return result.get(); + } + + if (result.isError()) { + // Do SERENITY_LOG when tag is available here. + } + + return defaultValue; + } + private: void registerProductForConsumption() { consumablesPerIteration += 1; diff --git a/src/tests/common/config_helper.hpp b/src/tests/common/config_helper.hpp index 7070db9..ab2c358 100644 --- a/src/tests/common/config_helper.hpp +++ b/src/tests/common/config_helper.hpp @@ -10,8 +10,8 @@ namespace serenity { namespace tests { inline SerenityConfig createAssuranceAnalyzerCfg( - const uint64_t windowSize, - const uint64_t maxCheckpoints, + const int64_t windowSize, + const int64_t maxCheckpoints, const double_t fractionalThreshold, const double_t severityLvl = detector::DEFAULT_SEVERITY_FRACTION, const double_t nearLvl = detector::DEFAULT_NEAR_FRACTION, @@ -19,16 +19,16 @@ inline SerenityConfig createAssuranceAnalyzerCfg( SerenityConfig cfg; cfg.set(detector::WINDOW_SIZE, windowSize); cfg.set(detector::MAX_CHECKPOINTS, maxCheckpoints); - cfg.set(detector::FRACTIONAL_THRESHOLD, fractionalThreshold); - cfg.set(detector::SEVERITY_FRACTION, severityLvl); - cfg.set(detector::NEAR_FRACTION, nearLvl); - cfg.set(detector::QUORUM, quorum); + cfg.set(detector::FRACTIONAL_THRESHOLD, fractionalThreshold); + cfg.set(detector::SEVERITY_FRACTION, severityLvl); + cfg.set(detector::NEAR_FRACTION, nearLvl); + cfg.set(detector::QUORUM, quorum); return cfg; } inline SerenityConfig createThresholdDetectorCfg( - const double_t utilization = detector::DEFAULT_UTILIZATION_THRESHOLD) { + const double_t utilization = detector::DEFAULT_UTILIZATION_THRESHOLD) { SerenityConfig cfg; cfg.set(detector::THRESHOLD, utilization); diff --git a/src/tests/serenity/config_test.cpp b/src/tests/serenity/config_test.cpp index 12dc367..c0bcf7c 100644 --- a/src/tests/serenity/config_test.cpp +++ b/src/tests/serenity/config_test.cpp @@ -20,10 +20,6 @@ const constexpr char* FIELD_BOOL = "FIELD_BOOL"; const constexpr bool DEFAULT_FIELD_BOOL = true; const constexpr bool MODIFIED_FIELD_BOOL = false; -const constexpr char* FIELD_UINT = "FIELD_UINT"; -const constexpr uint64_t DEFAULT_FIELD_UINT = 23424; -const constexpr uint64_t MODIFIED_FIELD_UINT = 3; - const constexpr char* FIELD_INT = "FIELD_INT"; const constexpr int64_t DEFAULT_FIELD_INT = -435; const constexpr int64_t MODIFIED_FIELD_INT = 34535; @@ -33,71 +29,68 @@ const constexpr double_t DEFAULT_FIELD_DOUBLE = 0.345345; const constexpr double_t MODIFIED_FIELD_DOUBLE = 3.432; -class TestConfigFilter { - public: - explicit TestConfigFilter(const SerenityConfig& customCfg) { - configure(customCfg); - } - - void configure(const SerenityConfig& externalConf) { - config = SerenityConfig(); - - config.set(FIELD_STR, (std::string)DEFAULT_FIELD_STR); - config.set(FIELD_BOOL, DEFAULT_FIELD_BOOL); - config.set(FIELD_UINT, DEFAULT_FIELD_UINT); - config.set(FIELD_INT, DEFAULT_FIELD_INT); - config.set(FIELD_DOUBLE, DEFAULT_FIELD_DOUBLE); - - config.applyConfig(externalConf); - } - +SerenityConfig loadSampleConfig() { SerenityConfig config; -}; + config.set(FIELD_STR, (std::string) MODIFIED_FIELD_STR); + config.set(FIELD_BOOL, MODIFIED_FIELD_BOOL); + config.set(FIELD_INT, MODIFIED_FIELD_INT); + config.set(FIELD_DOUBLE, MODIFIED_FIELD_DOUBLE); + return config; +} -TEST(SerenityConfigTest, DefaultValuesAvailable) { - // Create empty config with no configuration items. - SerenityConfig newConfig; +TEST(SerenityConfigTest, EmptyItemsTest) { + SerenityConfig config; + EXPECT_NONE(config.item(FIELD_STR)); + EXPECT_NONE(config.item(FIELD_BOOL)); + EXPECT_NONE(config.item(FIELD_INT)); + EXPECT_NONE(config.item(FIELD_DOUBLE)); +} - TestConfigFilter testFilter = TestConfigFilter(newConfig); - EXPECT_EQ(testFilter.config.item(FIELD_STR).get(), - (std::string) DEFAULT_FIELD_STR); - EXPECT_EQ(testFilter.config.item(FIELD_BOOL).get(), +TEST(SerenityConfigTest, DefaultItemsTest) { + SerenityConfig config; + EXPECT_EQ(config.item(FIELD_STR, DEFAULT_FIELD_STR), + DEFAULT_FIELD_STR); + EXPECT_EQ(config.item(FIELD_BOOL, DEFAULT_FIELD_BOOL), DEFAULT_FIELD_BOOL); - EXPECT_EQ(testFilter.config.item(FIELD_UINT).get(), - DEFAULT_FIELD_UINT); - EXPECT_EQ(testFilter.config.item(FIELD_INT).get(), + EXPECT_EQ(config.item(FIELD_INT, DEFAULT_FIELD_INT), DEFAULT_FIELD_INT); - EXPECT_EQ(testFilter.config.item(FIELD_DOUBLE).get(), + EXPECT_EQ(config.item(FIELD_DOUBLE, DEFAULT_FIELD_DOUBLE), DEFAULT_FIELD_DOUBLE); - - EXPECT_EQ(testFilter.config.item(FIELD_BOOL).get(), DEFAULT_FIELD_BOOL); } -TEST(SerenityConfigTest, ModifiedValuesAvailable) { - // Create config with custom configuration items. - SerenityConfig newConfig; - newConfig.set(FIELD_STR, (std::string)MODIFIED_FIELD_STR); - newConfig.set(FIELD_BOOL, MODIFIED_FIELD_BOOL); - newConfig.set(FIELD_UINT, MODIFIED_FIELD_UINT); - newConfig.set(FIELD_INT, MODIFIED_FIELD_INT); - newConfig.set(FIELD_DOUBLE, MODIFIED_FIELD_DOUBLE); +TEST(SerenityConfigTest, ModifiedItemsTest) { + SerenityConfig config; + EXPECT_EQ(config.item(FIELD_STR, DEFAULT_FIELD_STR), + DEFAULT_FIELD_STR); + EXPECT_EQ(config.item(FIELD_BOOL, DEFAULT_FIELD_BOOL), + DEFAULT_FIELD_BOOL); + EXPECT_EQ(config.item(FIELD_INT, DEFAULT_FIELD_INT), + DEFAULT_FIELD_INT); + EXPECT_EQ(config.item(FIELD_DOUBLE, DEFAULT_FIELD_DOUBLE), + DEFAULT_FIELD_DOUBLE); - TestConfigFilter testFilter = TestConfigFilter(newConfig); + config = loadSampleConfig(); - EXPECT_EQ(testFilter.config.item(FIELD_STR).get(), - (std::string) MODIFIED_FIELD_STR); - EXPECT_EQ(testFilter.config.item(FIELD_BOOL).get(), + EXPECT_EQ(config.item(FIELD_STR, DEFAULT_FIELD_STR), + MODIFIED_FIELD_STR); + EXPECT_EQ(config.item(FIELD_BOOL, DEFAULT_FIELD_BOOL), MODIFIED_FIELD_BOOL); - EXPECT_EQ(testFilter.config.item(FIELD_UINT).get(), - MODIFIED_FIELD_UINT); - EXPECT_EQ(testFilter.config.item(FIELD_INT).get(), + EXPECT_EQ(config.item(FIELD_INT, DEFAULT_FIELD_INT), MODIFIED_FIELD_INT); - EXPECT_EQ(testFilter.config.item(FIELD_DOUBLE).get(), + EXPECT_EQ(config.item(FIELD_DOUBLE, DEFAULT_FIELD_DOUBLE), MODIFIED_FIELD_DOUBLE); } +TEST(SerenityConfigTest, ErrorItemsTest) { + SerenityConfig config = loadSampleConfig(); + EXPECT_ERROR(config.item(FIELD_STR)); + EXPECT_ERROR(config.item(FIELD_BOOL)); + EXPECT_ERROR(config.item(FIELD_INT)); + EXPECT_ERROR(config.item(FIELD_DOUBLE)); +} + } // namespace tests } // namespace serenity } // namespace mesos From 3e297af27c6305b8786ed2b49652861cab58253b Mon Sep 17 00:00:00 2001 From: bplotka Date: Wed, 17 Feb 2016 20:52:39 +0100 Subject: [PATCH 4/7] Fixed small issue in signal detector. Signed-off-by: bplotka --- src/contention_detectors/signal_analyzers/drop.hpp | 5 ++--- src/tests/common/config_helper.hpp | 8 ++++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/contention_detectors/signal_analyzers/drop.hpp b/src/contention_detectors/signal_analyzers/drop.hpp index 82ea391..6610a14 100644 --- a/src/contention_detectors/signal_analyzers/drop.hpp +++ b/src/contention_detectors/signal_analyzers/drop.hpp @@ -59,7 +59,6 @@ class SignalDropAnalyzer : public SignalAnalyzer { : SignalAnalyzer(_tag), valueBeforeDrop(None()), quorumNum(0) { - setCfgWindowSize(_config.item( detector::WINDOW_SIZE, detector::DEFAULT_WINDOW_SIZE)); @@ -80,9 +79,9 @@ class SignalDropAnalyzer : public SignalAnalyzer { detector::NEAR_FRACTION, detector::DEFAULT_NEAR_FRACTION)); - setCfgNearFraction(_config.item( + setCfgSeverityFraction(_config.item( detector::SEVERITY_FRACTION, - detector::DEFAULT_NEAR_FRACTION)); + detector::DEFAULT_SEVERITY_FRACTION)); this->recalculateParams(); } diff --git a/src/tests/common/config_helper.hpp b/src/tests/common/config_helper.hpp index ab2c358..443eec4 100644 --- a/src/tests/common/config_helper.hpp +++ b/src/tests/common/config_helper.hpp @@ -19,10 +19,10 @@ inline SerenityConfig createAssuranceAnalyzerCfg( SerenityConfig cfg; cfg.set(detector::WINDOW_SIZE, windowSize); cfg.set(detector::MAX_CHECKPOINTS, maxCheckpoints); - cfg.set(detector::FRACTIONAL_THRESHOLD, fractionalThreshold); - cfg.set(detector::SEVERITY_FRACTION, severityLvl); - cfg.set(detector::NEAR_FRACTION, nearLvl); - cfg.set(detector::QUORUM, quorum); + cfg.set(detector::FRACTIONAL_THRESHOLD, fractionalThreshold); + cfg.set(detector::SEVERITY_FRACTION, severityLvl); + cfg.set(detector::NEAR_FRACTION, nearLvl); + cfg.set(detector::QUORUM, quorum); return cfg; } From df386ab21ab7b05abfbe57790f15246d577434c5 Mon Sep 17 00:00:00 2001 From: bplotka Date: Wed, 17 Feb 2016 21:14:14 +0100 Subject: [PATCH 5/7] Applied refactor to other components. Signed-off-by: bplotka --- src/contention_detectors/overload.cpp | 6 ++-- src/contention_detectors/overload.hpp | 2 +- src/filters/too_low_usage.hpp | 32 ++++++-------------- src/observers/strategies/cpu_contention.hpp | 33 +++++---------------- 4 files changed, 20 insertions(+), 53 deletions(-) diff --git a/src/contention_detectors/overload.cpp b/src/contention_detectors/overload.cpp index 2a641a4..02c6cba 100644 --- a/src/contention_detectors/overload.cpp +++ b/src/contention_detectors/overload.cpp @@ -24,7 +24,7 @@ void OverloadDetector::allProductsReady() { if (totalAgentCpus.isNone()) { SERENITY_LOG(ERROR) << std::string(NAME) - << " No total cpus in ResourceUsage"; + << " No total cpus in ResourceUsage"; produce(product); } @@ -34,7 +34,7 @@ void OverloadDetector::allProductsReady() { for (const ResourceUsage_Executor& inExec : usage.executors()) { // Validate for statistics and executor info. - if (!validate(inExec)) { + if (!hasRequiredFields(inExec)) { continue; } @@ -71,7 +71,7 @@ void OverloadDetector::allProductsReady() { produce(product); } -bool OverloadDetector::validate(const ResourceUsage_Executor& inExec) { +bool OverloadDetector::hasRequiredFields(const ResourceUsage_Executor& inExec) { if (!inExec.has_executor_info()) { SERENITY_LOG(ERROR) << "Executor " << " does not include executor_info"; diff --git a/src/contention_detectors/overload.hpp b/src/contention_detectors/overload.hpp index 077dbc5..4f21118 100644 --- a/src/contention_detectors/overload.hpp +++ b/src/contention_detectors/overload.hpp @@ -53,7 +53,7 @@ class OverloadDetector : protected: void allProductsReady() override; - bool validate(const ResourceUsage_Executor& inExec); + bool hasRequiredFields(const ResourceUsage_Executor& inExec); const Tag tag; const lambda::function cpuUsageGetFunction; diff --git a/src/filters/too_low_usage.hpp b/src/filters/too_low_usage.hpp index ec826d2..a97a4fe 100644 --- a/src/filters/too_low_usage.hpp +++ b/src/filters/too_low_usage.hpp @@ -14,24 +14,6 @@ namespace mesos { namespace serenity { -class TooLowUsageFilterConfig : public SerenityConfig { - public: - TooLowUsageFilterConfig() { } - - explicit TooLowUsageFilterConfig(const SerenityConfig& customCfg) { - this->initDefaults(); - this->applyConfig(customCfg); - } - - void initDefaults() { - //! double_t - //! Minimal cpu usage - this->items[too_low_usage::MINIMAL_CPU_USAGE] = - too_low_usage::DEFAULT_MINIMAL_CPU_USAGE; - } -}; - - /** * Filter out PR executors with too low metrics. * Currently we filter out when CPU Usage is below specified threshold. @@ -44,12 +26,12 @@ class TooLowUsageFilter : explicit TooLowUsageFilter( Consumer* _consumer, - SerenityConfig _conf, + const SerenityConfig& _conf, const Tag& _tag = Tag(QOS_CONTROLLER, NAME)) : Producer(_consumer), tag(_tag) { - SerenityConfig config = TooLowUsageFilterConfig(_conf); - this->cfgMinimalCpuUsage = - config.item(too_low_usage::MINIMAL_CPU_USAGE).get(); + setCfgMinimalCpuUsage(_conf.item( + too_low_usage::MINIMAL_CPU_USAGE, + too_low_usage::DEFAULT_MINIMAL_CPU_USAGE)); } ~TooLowUsageFilter(); @@ -58,7 +40,11 @@ class TooLowUsageFilter : Try consume(const ResourceUsage& in); - public: + void setCfgMinimalCpuUsage(double_t cfgMinimalCpuUsage) { + TooLowUsageFilter::cfgMinimalCpuUsage = cfgMinimalCpuUsage; + } + + protected: const Tag tag; double_t cfgMinimalCpuUsage; diff --git a/src/observers/strategies/cpu_contention.hpp b/src/observers/strategies/cpu_contention.hpp index 4cb8867..b951cf4 100644 --- a/src/observers/strategies/cpu_contention.hpp +++ b/src/observers/strategies/cpu_contention.hpp @@ -12,30 +12,6 @@ namespace mesos { namespace serenity { -class CpuContentionStrategyConfig : public SerenityConfig { - public: - CpuContentionStrategyConfig() { - this->initDefaults(); - } - - explicit CpuContentionStrategyConfig(const SerenityConfig& customCfg) { - this->initDefaults(); - this->applyConfig(customCfg); - } - - void initDefaults() { - // uint64_t - // Specify the initial value of iterations we should wait until - // we create new correction. - this->items[strategy::CONTENTION_COOLDOWN] = - strategy::DEFAULT_CONTENTION_COOLDOWN; - // double_t - this->items[strategy::DEFAULT_CPU_SEVERITY] = - strategy::DEFAULT_DEFAULT_CPU_SEVERITY; - } -}; - - /** * Checks contentions and choose executors to kill. * It accepts only Contention_Type_CPU. @@ -51,7 +27,9 @@ class CpuContentionStrategy : public RevocationStrategy { const lambda::function& _cpuUsageGetFunction) : RevocationStrategy(Tag(QOS_CONTROLLER, "CpuContentionStrategy")), getCpuUsage(_cpuUsageGetFunction) { - SerenityConfig config = CpuContentionStrategyConfig(_config); + setDefaultSeverity(_config.item( + strategy::DEFAULT_CPU_SEVERITY, + strategy::DEFAULT_DEFAULT_CPU_SEVERITY)); } Try decide(ExecutorAgeFilter* ageFilter, @@ -60,11 +38,14 @@ class CpuContentionStrategy : public RevocationStrategy { static const constexpr char* NAME = "CpuContentionStrategy"; + void setDefaultSeverity(double_t defaultSeverity) { + CpuContentionStrategy::defaultSeverity = defaultSeverity; + } + private: const lambda::function getCpuUsage; // cfg parameters. - uint64_t cooldownTime; double_t defaultSeverity; }; From 1a7e32a22de6fc4b4caeea2f17046b4f39f9f884 Mon Sep 17 00:00:00 2001 From: bplotka Date: Fri, 19 Feb 2016 16:35:27 +0100 Subject: [PATCH 6/7] First part of Serenity Config and validation refactor. Signed-off-by: bplotka --- src/contention_detectors/overload.cpp | 2 - src/contention_detectors/overload.hpp | 5 +- .../signal_analyzers/drop.cpp | 25 +- .../signal_analyzers/drop.hpp | 115 +++---- src/filters/too_low_usage.hpp | 2 +- .../qos_controller/serenity_controller.cpp | 8 +- .../qos_controller/serenity_controller.hpp | 11 +- .../serenity_controller_module.cpp | 4 +- src/observers/strategies/cpu_contention.hpp | 2 +- src/observers/strategies/seniority.hpp | 14 +- src/pipeline/qos_pipeline.hpp | 16 +- src/serenity/config.hpp | 296 +++++++++++------- src/serenity/default_vars.hpp | 14 +- src/serenity/serenity.hpp | 16 - .../contention_detectors/overload_test.cpp | 6 +- src/tests/serenity/config_test.cpp | 98 ++++-- 16 files changed, 354 insertions(+), 280 deletions(-) diff --git a/src/contention_detectors/overload.cpp b/src/contention_detectors/overload.cpp index 02c6cba..178ca03 100644 --- a/src/contention_detectors/overload.cpp +++ b/src/contention_detectors/overload.cpp @@ -75,7 +75,6 @@ bool OverloadDetector::hasRequiredFields(const ResourceUsage_Executor& inExec) { if (!inExec.has_executor_info()) { SERENITY_LOG(ERROR) << "Executor " << " does not include executor_info"; - // Filter out these executors. return false; } @@ -83,7 +82,6 @@ bool OverloadDetector::hasRequiredFields(const ResourceUsage_Executor& inExec) { SERENITY_LOG(ERROR) << "Executor " << inExec.executor_info().executor_id().value() << " does not include statistics."; - // Filter out these executors. return false; } diff --git a/src/contention_detectors/overload.hpp b/src/contention_detectors/overload.hpp index 4f21118..1992e8e 100644 --- a/src/contention_detectors/overload.hpp +++ b/src/contention_detectors/overload.hpp @@ -39,8 +39,9 @@ class OverloadDetector : Producer(_consumer) { // Parse config values. setCfgUtilizationThreshold( - _conf.item(detector::THRESHOLD, - detector::DEFAULT_UTILIZATION_THRESHOLD)); + _conf.getItemOrDefault( + detector::THRESHOLD, + detector::DEFAULT_UTILIZATION_THRESHOLD)); } ~OverloadDetector() {} diff --git a/src/contention_detectors/signal_analyzers/drop.cpp b/src/contention_detectors/signal_analyzers/drop.cpp index 0fac13a..75bdb6e 100644 --- a/src/contention_detectors/signal_analyzers/drop.cpp +++ b/src/contention_detectors/signal_analyzers/drop.cpp @@ -1,3 +1,4 @@ +#include #include #include @@ -22,8 +23,7 @@ void SignalDropAnalyzer::recalculateParams() { this->window.clear(); this->basePoints.clear(); - uint64_t windowSize = this->cfgWindowSize; - + uint64_t windowSize = cfgWindowSize; // Find the biggest n in the T-2^n which fits within window length. uint64_t checkpoints = 0; while (windowSize > 0) { @@ -31,34 +31,31 @@ void SignalDropAnalyzer::recalculateParams() { checkpoints++; } - // Make sure it does not exceed MAX_CHECKPOINSs option. - if (checkpoints > this->cfgMaxCheckpoints) { - checkpoints = this->cfgMaxCheckpoints; - } + // Make sure it does not exceed MAX_CHECKPOINTS option. + checkpoints = min(checkpoints, cfgMaxCheckpoints); // Get the Quorum number from QUORUM fraction parameter. - this->quorumNum = this->cfgQuroum * checkpoints; - if (this->quorumNum == 0 || this->quorumNum > checkpoints) { + quorumNum = cfgQuroumFraction * checkpoints; + if (quorumNum == 0 || quorumNum > checkpoints) { SERENITY_LOG(WARNING) << "Bad value for Quorum parameter. Creating 100%" << " quorum."; - this->quorumNum = checkpoints; + quorumNum = checkpoints; } std::stringstream checkpointLog; - checkpointLog << "Assurance Parameters: Quorum = " - << this->quorumNum << "/" + checkpointLog << "Assurance Parameters: Quorum = " << quorumNum << "/" << checkpoints << " Checkpoints [ "; // Iterate over window and initialize it. Choose proper base points starting // from the end of window. uint64_t choosenNum = pow(2, (--checkpoints)); - for (uint64_t i = this->cfgWindowSize; i > 0 ; i--) { - this->window.push_back(detector::DEFAULT_START_VALUE); + for (uint64_t i = cfgWindowSize; i > 0 ; i--) { + window.push_back(detector::DEFAULT_START_VALUE); if (choosenNum == i) { checkpointLog << "T-" << choosenNum << " "; choosenNum /= 2; - basePoints.push_back(--this->window.end()); + basePoints.push_back(--window.end()); } } checkpointLog << "]"; diff --git a/src/contention_detectors/signal_analyzers/drop.hpp b/src/contention_detectors/signal_analyzers/drop.hpp index 6610a14..775be6f 100644 --- a/src/contention_detectors/signal_analyzers/drop.hpp +++ b/src/contention_detectors/signal_analyzers/drop.hpp @@ -14,7 +14,6 @@ #include "messages/serenity.hpp" #include "serenity/config.hpp" -#include "serenity/default_vars.hpp" #include "serenity/data_utils.hpp" #include "serenity/executor_map.hpp" #include "serenity/executor_set.hpp" @@ -28,8 +27,6 @@ namespace mesos { namespace serenity { -#define SIGNAL_DROP_ANALYZER_NAME "AssuranceDropAnalyzer" - /** * Dynamic implementation of sequential change point detection. * @@ -59,29 +56,30 @@ class SignalDropAnalyzer : public SignalAnalyzer { : SignalAnalyzer(_tag), valueBeforeDrop(None()), quorumNum(0) { - setCfgWindowSize(_config.item( - detector::WINDOW_SIZE, - detector::DEFAULT_WINDOW_SIZE)); - setCfgMaxCheckpoints(_config.item( - detector::MAX_CHECKPOINTS, - detector::DEFAULT_MAX_CHECKPOINTS)); + setWindowsSizeAndMaxCheckpoints( + _config.getItemAndSetDefault( + SignalDropAnalyzer::WINDOW_SIZE_KEY, + SignalDropAnalyzer::WINDOW_SIZE_DEFAULT), + _config.getItemAndSetDefault( + SignalDropAnalyzer::MAX_CHECKPOINTS_KEY, + SignalDropAnalyzer::MAX_CHECKPOINTS_DEFAULT)); - setCfgQuroum(_config.item( - detector::QUORUM, - detector::DEFAULT_QUORUM)); + setQuroumFraction(_config.getItemAndSetDefault( + SignalDropAnalyzer::QUORUM_FRACTION_KEY, + SignalDropAnalyzer::QUORUM_FRACTION_DEFAULT)); - setCfgFractionalThreshold(_config.item( - detector::FRACTIONAL_THRESHOLD, - detector::DEFAULT_FRACTIONAL_THRESHOLD)); + setFractionalThreshold(_config.getItemAndSetDefault( + SignalDropAnalyzer::FRACTIONAL_THRESHOLD_KEY, + SignalDropAnalyzer::FRACTIONAL_THRESHOLD_DEFAULT)); - setCfgNearFraction(_config.item( - detector::NEAR_FRACTION, - detector::DEFAULT_NEAR_FRACTION)); + setNearFraction(_config.getItemAndSetDefault( + SignalDropAnalyzer::NEAR_FRACTION_KEY, + SignalDropAnalyzer::NEAR_FRACTION_DEFAULT)); - setCfgSeverityFraction(_config.item( - detector::SEVERITY_FRACTION, - detector::DEFAULT_SEVERITY_FRACTION)); + setSeverityFraction(_config.getItemAndSetDefault( + SignalDropAnalyzer::SEVERITY_FRACTION_KEY, + SignalDropAnalyzer::SEVERITY_FRACTION_DEFAULT)); this->recalculateParams(); } @@ -97,71 +95,80 @@ class SignalDropAnalyzer : public SignalAnalyzer { */ void shiftBasePoints(); - //! int64_t - //! How far in the past we look. - void setCfgWindowSize(int64_t cfgWindowSize) { - SignalDropAnalyzer::cfgWindowSize = cfgWindowSize; - } + static const constexpr char* WINDOW_SIZE_KEY = "WINDOW_SIZE"; + static const constexpr char* FRACTIONAL_THRESHOLD_KEY = + "FRACTIONAL_THRESHOLD"; + static const constexpr char* SEVERITY_FRACTION_KEY = "SEVERITY_FRACTION"; + static const constexpr char* NEAR_FRACTION_KEY = "NEAR_FRACTION"; + static const constexpr char* MAX_CHECKPOINTS_KEY = "MAX_CHECKPOINTS"; + static const constexpr char* QUORUM_FRACTION_KEY = "QUORUM_FRACTION"; + +protected: + void recalculateParams(); - //! int64_t - //! Maximum number of checkpoints we will have in our assurance detector. + //! WindowSize: How far in the past we look. + //! MaxCheckpoints: Maximum number of checkpoints we will have in our + //! assurance detector. //! Checkpoints are the reference assurance_test(base) points which we refer //! to in the past when detecting drop or not. - //! It needs to be 0 < < WINDOW_SIZE - void setCfgMaxCheckpoints(int64_t cfgMaxCheckpoints) { - SignalDropAnalyzer::cfgMaxCheckpoints = cfgMaxCheckpoints; + //! It needs to be 0 < and < WINDOW_SIZE. + void setWindowsSizeAndMaxCheckpoints( + SerenityItem _cfgWindowSize, + SerenityItem _cfgMaxCheckpoints) { + cfgWindowSize = _cfgWindowSize.validateValueIsPositive().getValueOrDefault(); + cfgMaxCheckpoints = _cfgMaxCheckpoints + .validateValueIsPositive() + .validateValueIsBelow(_cfgWindowSize) + .getValueOrDefault(); } - //! double_t - //! Fraction of checkpoints' votes that important decision needs to obtain. - void setCfgQuroum(double_t cfgQuroum) { - SignalDropAnalyzer::cfgQuroum = cfgQuroum; + //! Fraction of checkpoints' votes needed to make a Drop contention. + void setQuroumFraction(SerenityItem item) { + cfgQuroumFraction = item.validateValueIsPositive().getValueOrDefault(); } - //! double_t //! Defines how much (relatively to base point) value must drop to trigger //! contention. - //! Most signal_analyzer will use that. - void setCfgFractionalThreshold(double_t cfgFractionalThreshold) { - SignalDropAnalyzer::cfgFractionalThreshold = cfgFractionalThreshold; + void setFractionalThreshold(SerenityItem item) { + cfgFractionalThreshold = item.validateValueIsPositive().getValueOrDefault(); } - //! double_t //! You can adjust how big severity is created for a defined drop. //! if -1 then unknown severity will be reported. - void setCfgSeverityFraction(double_t cfgSeverityFraction) { - SignalDropAnalyzer::cfgSeverityFraction = cfgSeverityFraction; + void setSeverityFraction(SerenityItem item) { + cfgSeverityFraction = item.getValueOrDefault(); } - //! double_t //! Tolerance fraction of threshold if signal is accepted as returned to //! previous state after drop. - void setCfgNearFraction(double_t cfgNearFraction) { - SignalDropAnalyzer::cfgNearFraction = cfgNearFraction; + void setNearFraction(SerenityItem item) { + cfgNearFraction = item.validateValueIsPositive().getValueOrDefault(); } - protected: std::list window; std::list::iterator> basePoints; // If none then there was no drop. Option valueBeforeDrop; - uint32_t dropVotes; - uint32_t quorumNum; + uint64_t dropVotes; + uint64_t quorumNum; - // cfg parameters. + // Cfg parameters. int64_t cfgWindowSize; int64_t cfgMaxCheckpoints; - double_t cfgQuroum; + double_t cfgQuroumFraction; double_t cfgFractionalThreshold; double_t cfgSeverityFraction; double_t cfgNearFraction; - /** - * It is possible to dynamically change analyzer configuration. - */ - void recalculateParams(); + // Cfg default values. + static const constexpr int64_t WINDOW_SIZE_DEFAULT = 10; + static constexpr double_t FRACTIONAL_THRESHOLD_DEFAULT = 0.3; + static constexpr double_t SEVERITY_FRACTION_DEFAULT = 2.1; + static constexpr double_t NEAR_FRACTION_DEFAULT = 0.1; + static constexpr int64_t MAX_CHECKPOINTS_DEFAULT = 3; + static constexpr double_t QUORUM_FRACTION_DEFAULT = 0.7; }; diff --git a/src/filters/too_low_usage.hpp b/src/filters/too_low_usage.hpp index a97a4fe..bf44b32 100644 --- a/src/filters/too_low_usage.hpp +++ b/src/filters/too_low_usage.hpp @@ -29,7 +29,7 @@ class TooLowUsageFilter : const SerenityConfig& _conf, const Tag& _tag = Tag(QOS_CONTROLLER, NAME)) : Producer(_consumer), tag(_tag) { - setCfgMinimalCpuUsage(_conf.item( + setCfgMinimalCpuUsage(_conf.getItemOrDefault( too_low_usage::MINIMAL_CPU_USAGE, too_low_usage::DEFAULT_MINIMAL_CPU_USAGE)); } diff --git a/src/mesos_modules/qos_controller/serenity_controller.cpp b/src/mesos_modules/qos_controller/serenity_controller.cpp index 1678130..26e554e 100644 --- a/src/mesos_modules/qos_controller/serenity_controller.cpp +++ b/src/mesos_modules/qos_controller/serenity_controller.cpp @@ -32,10 +32,10 @@ class SerenityControllerProcess : public: SerenityControllerProcess( const lambda::function()>& _usage, - std::shared_ptr _pipeline, + std::unique_ptr _pipeline, double _onEmptyCorrectionInterval) : usage(_usage), - pipeline(_pipeline), + pipeline(std::move(_pipeline)), onEmptyCorrectionInterval(_onEmptyCorrectionInterval) {} Future corrections() { @@ -84,7 +84,7 @@ class SerenityControllerProcess : private: const lambda::function()> usage; - std::shared_ptr pipeline; + std::unique_ptr pipeline; //! How much time we wait in case of empty correction. //! This value should be near the perf interval since it is useless //! to rerun QoS pipeline on the same perf's counter collection. @@ -109,7 +109,7 @@ Try SerenityController::initialize( } process.reset(new SerenityControllerProcess( - usage, this->pipeline, this->onEmptyCorrectionInterval)); + usage, std::move(this->pipeline), this->onEmptyCorrectionInterval)); spawn(process.get()); return Nothing(); diff --git a/src/mesos_modules/qos_controller/serenity_controller.hpp b/src/mesos_modules/qos_controller/serenity_controller.hpp index acf6cea..c31ff6b 100644 --- a/src/mesos_modules/qos_controller/serenity_controller.hpp +++ b/src/mesos_modules/qos_controller/serenity_controller.hpp @@ -26,15 +26,16 @@ class SerenityControllerProcess; class SerenityController: public slave::QoSController { public: explicit SerenityController( - std::shared_ptr _pipeline, + std::unique_ptr _pipeline, double _onEmptyCorrectionInterval) - : pipeline(_pipeline), + : pipeline(std::move(_pipeline)), onEmptyCorrectionInterval(_onEmptyCorrectionInterval) {} static Try create( - std::shared_ptr _pipeline, + std::unique_ptr _pipeline, double _onEmptyCorrectionInterval = 5) { - return new SerenityController(_pipeline, _onEmptyCorrectionInterval); + return new SerenityController(std::move(_pipeline), + _onEmptyCorrectionInterval); } virtual ~SerenityController(); @@ -46,7 +47,7 @@ class SerenityController: public slave::QoSController { protected: process::Owned process; - std::shared_ptr pipeline; + std::unique_ptr pipeline; double onEmptyCorrectionInterval; }; diff --git a/src/mesos_modules/qos_controller/serenity_controller_module.cpp b/src/mesos_modules/qos_controller/serenity_controller_module.cpp index fef8c19..fceb134 100644 --- a/src/mesos_modules/qos_controller/serenity_controller_module.cpp +++ b/src/mesos_modules/qos_controller/serenity_controller_module.cpp @@ -43,12 +43,12 @@ static QoSController* createSerenityController( SerenityConfig conf; double onEmptyCorrectionInterval = - conf.item(ON_EMPTY_CORRECTION_INTERVAL, + conf.getItemOrDefault(ON_EMPTY_CORRECTION_INTERVAL, DEFAULT_ON_EMPTY_CORRECTION_INTERVAL); // Use static constructor of QoSController. Try result = - SerenityController::create(std::shared_ptr( + SerenityController::create(std::unique_ptr( new CpuQoSPipeline(conf)), onEmptyCorrectionInterval); diff --git a/src/observers/strategies/cpu_contention.hpp b/src/observers/strategies/cpu_contention.hpp index b951cf4..2ed4a4b 100644 --- a/src/observers/strategies/cpu_contention.hpp +++ b/src/observers/strategies/cpu_contention.hpp @@ -27,7 +27,7 @@ class CpuContentionStrategy : public RevocationStrategy { const lambda::function& _cpuUsageGetFunction) : RevocationStrategy(Tag(QOS_CONTROLLER, "CpuContentionStrategy")), getCpuUsage(_cpuUsageGetFunction) { - setDefaultSeverity(_config.item( + setDefaultSeverity(_config.getItemOrDefault( strategy::DEFAULT_CPU_SEVERITY, strategy::DEFAULT_DEFAULT_CPU_SEVERITY)); } diff --git a/src/observers/strategies/seniority.hpp b/src/observers/strategies/seniority.hpp index 7a0518c..f204323 100644 --- a/src/observers/strategies/seniority.hpp +++ b/src/observers/strategies/seniority.hpp @@ -20,20 +20,14 @@ namespace serenity { */ class SeniorityStrategy : public RevocationStrategy { public: - SeniorityStrategy() : RevocationStrategy(Tag(QOS_CONTROLLER, NAME)) { - initialize(); - } - /** * TODO(skonefal): SerenityConfig should have const methods inside. * Currently, it cannot be passed as const. */ explicit SeniorityStrategy(SerenityConfig _config) : RevocationStrategy(Tag(QOS_CONTROLLER, NAME)) { - initialize(); - if (_config.hasKey(STARTING_SEVERITY_KEY)) { - severity = _config.item(STARTING_SEVERITY_KEY).get(); - } + severity = _config.getItemOrDefault(STARTING_SEVERITY_KEY, + DEFAULT_SEVERITY); } Try decide(ExecutorAgeFilter*, @@ -44,10 +38,6 @@ class SeniorityStrategy : public RevocationStrategy { static const constexpr char* NAME = "SeniorityStrategy"; private: - void initialize() { - severity = DEFAULT_SEVERITY; - } - static const constexpr double_t DEFAULT_SEVERITY = 0.1; double_t severity; diff --git a/src/pipeline/qos_pipeline.hpp b/src/pipeline/qos_pipeline.hpp index 6b0bd46..6c93321 100644 --- a/src/pipeline/qos_pipeline.hpp +++ b/src/pipeline/qos_pipeline.hpp @@ -126,37 +126,39 @@ class CpuQoSPipeline : public QoSControllerPipeline { ipcDropDetector( &cacheOccupancyContentionObserver, usage::getEmaIpc, - conf[SIGNAL_DROP_ANALYZER_NAME], + conf.getSectionOrNew(SIGNAL_DROP_ANALYZER_NAME), Tag(QOS_CONTROLLER, "IPC detectorFilter"), Contention_Type_IPC), ipcEMAFilter( &ipcDropDetector, usage::getIpc, usage::setEmaIpc, - conf.item(ema::ALPHA_IPC, ema::DEFAULT_ALPHA_IPC), + conf.getItemOrDefault(ema::ALPHA_IPC, + ema::DEFAULT_ALPHA_IPC), Tag(QOS_CONTROLLER, "ipcEMAFilter")), tooLowUsageFilter( &ipcEMAFilter, - conf[TooLowUsageFilter::NAME], + conf.getSectionOrNew(TooLowUsageFilter::NAME), Tag(QOS_CONTROLLER, "tooLowCPUUsageFilter")), cpuContentionObserver( &correctionMerger, &ageFilter, new CpuContentionStrategy( - conf[CpuContentionStrategy::NAME], + conf.getSectionOrNew(CpuContentionStrategy::NAME), usage::getEmaCpuUsage), strategy::DEFAULT_CONTENTION_COOLDOWN, Tag(QOS_CONTROLLER, CpuContentionStrategy::NAME)), overloadDetector( &cpuContentionObserver, usage::getEmaCpuUsage, - conf[OverloadDetector::NAME], + conf.getSectionOrNew(OverloadDetector::NAME), Tag(QOS_CONTROLLER, "CPU High Usage utilization detector")), cpuEMAFilter( &overloadDetector, usage::getCpuUsage, usage::setEmaCpuUsage, - conf.item(ema::ALPHA_CPU, ema::DEFAULT_ALPHA_CPU), + conf.getItemOrDefault(ema::ALPHA_CPU, + ema::DEFAULT_ALPHA_CPU), Tag(QOS_CONTROLLER, "cpuEMAFilter")), cumulativeFilter( &tooLowUsageFilter, @@ -164,7 +166,7 @@ class CpuQoSPipeline : public QoSControllerPipeline { // First item in pipeline. For now, close the pipeline for QoS. valveFilter( &cumulativeFilter, - conf.item(VALVE_OPENED, DEFAULT_VALVE_OPENED), + conf.getItemOrDefault(VALVE_OPENED, DEFAULT_VALVE_OPENED), Tag(QOS_CONTROLLER, "valveFilter")) { this->ageFilter.addConsumer(&valveFilter); // Setup starting producer. diff --git a/src/serenity/config.hpp b/src/serenity/config.hpp index d75bf4e..22b603c 100644 --- a/src/serenity/config.hpp +++ b/src/serenity/config.hpp @@ -17,124 +17,175 @@ namespace mesos { namespace serenity { -/** - * Serenity Config class which implements basic mechanism - * to support specifying config parameters via string key map & sections. - * - * Check config_test.cpp to see example usage. - */ -class SerenityConfig { +template +class SerenityItem { public: - SerenityConfig() {} + SerenityItem(T _value, T _defaultValue, std::string _key) : + value(_value), defaultValue(_defaultValue), key(_key) {}; - /** - * Variant type for storing multiple types of data in configuration. - */ - using CfgVariant = boost::variant< - bool, int64_t, double_t, std::string>; + T getValueOrDefault() { + if (validationFailed) { + return defaultValue; + } - /** - * Overlapping custom configuration options using recursive copy. - */ - void applyConfig(const SerenityConfig& customCfg) { - recursiveCfgCopy(this, customCfg); + return value; } - /** - * Templated, safe getter for item in config. - */ - template - const Result item(const std::string& key) const { - static_assert(std::is_same() - || std::is_same() - || std::is_same() - || std::is_same(), - "T must be one of the types stored in CfgVariant (bool, " - "int64_t, double_t, string)"); + //! Validates that value is below given threshold item. + SerenityItem& validateValueIsBelow(SerenityItem _thresholdItem) { + assertTypeMatchNumericVariant(); - Result result = None(); + if (value > _thresholdItem.getValueOrDefault()) { + LOG(WARNING) << key << " option which is " << value + << "must be below the " << _thresholdItem.key + << " which is " << _thresholdItem.getValueOrDefault(); - // Get item from items map. - Option variantResult = getItem(key); + validationFailed = true; + } - if (variantResult.isSome()) { - // When item is found, try to parse it to the specified T type. - try { - result = boost::get(variantResult.get()); - } catch (std::exception& e) { - LOG(ERROR) << "Failed to parse " << key - << " field: " << e.what(); - result = Result::error(e.what()); - } + return *this; + } + + //! Validates that value is below given threshold. + SerenityItem& validateValueIsBelow(T thresholdValue) { + assertTypeMatchNumericVariant(); + + if (value > thresholdValue) { + LOG(WARNING) << key << " option which is " << value + << "must be below " << thresholdValue; + + validationFailed = true; + } + + return *this; + } + + //! Validates that value is above given threshold item. + SerenityItem& validateValueIsAbove(SerenityItem _thresholdItem) { + assertTypeMatchNumericVariant(); + + if (value < _thresholdItem.getValueOrDefault()) { + LOG(WARNING) << key << " option which is " << value + << "must be above the " << _thresholdItem.key + << " which is " << _thresholdItem.getValueOrDefault(); + + validationFailed = true; + } + + return *this; + } + + //! Validates that value is above given threshold. + SerenityItem& validateValueIsAbove(T thresholdValue) { + assertTypeMatchNumericVariant(); + + if (value < thresholdValue) { + LOG(WARNING) << key << " option which is " << value + << "must be above " << thresholdValue; + + validationFailed = true; } - return result; + return *this; + } + + //! Validates that value is positive. + SerenityItem& validateValueIsPositive() { + assertTypeMatchNumericVariant(); + + if (value < 0) { + LOG(WARNING) << key << " option which is " << value + << "must be above 0"; + + validationFailed = true; + } + + return *this; + } + + protected: + T value; + const T defaultValue; + const std::string key; + bool validationFailed = false; + + private: + template + static void assertTypeMatchNumericVariant() const { + static_assert(std::is_same() + || std::is_same(), + "Function supports only following numeric types: int64_t, " + "double_t"); } +}; + + +/** + * Serenity Config class which implements basic mechanism + * to support specifying config parameters via string key map & sections. + * + * Check config_test.cpp to see example usage. + */ +class SerenityConfig { + public: + SerenityConfig() {} /** - * Templated, safe getter for item in config. Sets default value in case of - * error or none. + * Safe getter for item in config. + * Return default value in case of error or none. */ template - const T item(const std::string& key, T defaultValue) const { - static_assert(std::is_same() - || std::is_same() - || std::is_same() - || std::is_same(), - "T must be one of the types stored in CfgVariant (bool, " - "int64_t, double_t, string)"); + const SerenityItem getItemAndSetDefault( + const std::string& key, T defaultValue) const { + assertTypeMatchVariant(); + T result = defaultValue; // Get item from items map. - Option variantResult = getItem(key); + Option value = getVariantValue(key); - if (variantResult.isSome()) { + if (value.isSome()) { // When item is found, try to parse it to the specified T type. try { - result = boost::get(variantResult.get()); + result = boost::get(value.get()); } catch (std::exception& e) { LOG(ERROR) << "Failed to parse " << key << " to type " << typeid(T).name() << ". Field: " << e.what(); } } - return result; + return SerenityItem(result, defaultValue, key); } /** * Gets config section. - * In case there is not one, create empty section. */ - SerenityConfig& operator[](const std::string& key) { - return *getSection(key); - } + Option getSection(const std::string& sectionKey) { + auto mapItem = this->sections.find(sectionKey); + if (mapItem != this->sections.end()) { + return *(mapItem->second); + } - /** - * Templated set config value. - */ - template - void set(const std::string& key, T value) { - static_assert(std::is_same() - || std::is_same() - || std::is_same() - || std::is_same(), - "T must be one of the types stored in CfgVariant (bool, " - "int64_t, double_t, string)"); - this->setVariant(key, value); + // Element not found. + return None(); } /** - * Templated set config value for char*. + * Gets config section. + * In case there is not one, create empty section. */ - void set(const std::string& key, char* value) { - this->setVariant(key, (std::string) value); - } + SerenityConfig& getSectionOrNew(const std::string& sectionKey) { + auto mapItem = this->sections.find(sectionKey); + if (mapItem != this->sections.end()) { + return *(mapItem->second); + } - /** - * Sets CfgVariant config value. - */ - void setVariant(const std::string& key, SerenityConfig::CfgVariant value) { - this->items[key] = value; + // In case of no section under this key - create empty section. + std::shared_ptr newSection = + std::shared_ptr(new SerenityConfig()); + sections[sectionKey] = newSection; + + return *newSection; } bool hasKey(const std::string& key) const { @@ -142,18 +193,22 @@ class SerenityConfig { } /** - * TODO(skonefal): Add UT for usage of this enum. - */ - enum ConfigurationType : int { - BOOL = 0, - INT64 = 1, - UINT64 = 2, - DOUBLE = 3, - STRING = 4 - }; + * Overlapping custom configuration options using recursive copy. + */ + void applyConfig(const SerenityConfig& customCfg) { + recursiveCfgCopy(this, customCfg); + } protected: - std::unordered_map items; + /** + * Variant type for storing multiple types of data in configuration. + */ + using Item = boost::variant; + + /** + * Item + */ + std::unordered_map items; /** * Support for hierarchical configuration sections. @@ -161,33 +216,25 @@ class SerenityConfig { std::unordered_map> sections; /** - * Getter for section. - * In case of no section - create such. + * Put config value for Item types. */ - std::shared_ptr getSection(const std::string& sectionKey) { - auto mapItem = this->sections.find(sectionKey); - if (mapItem != this->sections.end()) { - return mapItem->second; - } - - // In case of no section under this key - create empty section. - auto newSection = std::make_shared(SerenityConfig()); - this->sections[sectionKey] = newSection; - - return newSection; + template + void put(const std::string& key, T value) { + this->putVariant(key, value); } /** - * Getter for field. + * Put config value for char*. */ - Option getItem(const std::string& itemKey) const { - auto mapItem = this->items.find(itemKey); - if (mapItem != this->items.end()) { - return mapItem->second; - } + void put(const std::string& key, char* value) { + this->putVariant(key, (std::string) value); + } - // Element not found. - return None(); + /** + * Put Item config value. + */ + void putVariant(const std::string& key, SerenityConfig::Item value) { + this->items[key] = value; } /** @@ -201,9 +248,34 @@ class SerenityConfig { for (auto customSection : customCfg.sections) { this->recursiveCfgCopy( - &((*base)[customSection.first]), *customSection.second); + &(base->getSectionOrNew(customSection.first)), *customSection.second); } } + + /** + * Getter for Item. + */ + Option getVariantValue( + const std::string& itemKey) const { + auto mapItem = this->items.find(itemKey); + if (mapItem != this->items.end()) { + return mapItem->second; + } + + // Element not found. + return None(); + } + + private: + template + static void assertTypeMatchVariant() const { + static_assert(std::is_same() + || std::is_same() + || std::is_same() + || std::is_same(), + "Config supports only following types: bool, int64_t, " + "double_t, string"); + } }; diff --git a/src/serenity/default_vars.hpp b/src/serenity/default_vars.hpp index 03fb202..22f4185 100644 --- a/src/serenity/default_vars.hpp +++ b/src/serenity/default_vars.hpp @@ -38,20 +38,8 @@ constexpr double_t DEFAULT_ALPHA_IPC = 0.9; } // namespace ema namespace detector { -const constexpr char* ANALYZER_TYPE = "ANALYZER_TYPE"; -const constexpr char* WINDOW_SIZE = "WINDOW_SIZE"; -constexpr int64_t DEFAULT_WINDOW_SIZE = 10; -const constexpr char* FRACTIONAL_THRESHOLD = "FRACTIONAL_THRESHOLD"; -constexpr double_t DEFAULT_FRACTIONAL_THRESHOLD = 0.3; -const constexpr char* SEVERITY_FRACTION = "SEVERITY_FRACTION"; -constexpr double_t DEFAULT_SEVERITY_FRACTION = 2.1; -const constexpr char* NEAR_FRACTION = "NEAR_FRACTION"; -constexpr double_t DEFAULT_NEAR_FRACTION = 0.1; -const constexpr char* MAX_CHECKPOINTS = "MAX_CHECKPOINTS"; -constexpr int64_t DEFAULT_MAX_CHECKPOINTS = 3; -const constexpr char* QUORUM = "QUORUM"; -constexpr double_t DEFAULT_QUORUM = 0.70; +//const constexpr char* ANALYZER_TYPE = "ANALYZER_TYPE"; constexpr double_t DEFAULT_START_VALUE = 0.00001; const constexpr char* THRESHOLD = "THRESHOLD"; diff --git a/src/serenity/serenity.hpp b/src/serenity/serenity.hpp index f777e7b..58b2089 100644 --- a/src/serenity/serenity.hpp +++ b/src/serenity/serenity.hpp @@ -35,22 +35,6 @@ class BaseFilter { virtual ~BaseFilter() {} - /** - * Getting Result safely with default value when in error or none state. - */ - template - T safeGetResult(Result result, T defaultValue) { - if (result.isSome()) { - return result.get(); - } - - if (result.isError()) { - // Do SERENITY_LOG when tag is available here. - } - - return defaultValue; - } - private: void registerProductForConsumption() { consumablesPerIteration += 1; diff --git a/src/tests/contention_detectors/overload_test.cpp b/src/tests/contention_detectors/overload_test.cpp index 6b22a0e..d60b8fe 100644 --- a/src/tests/contention_detectors/overload_test.cpp +++ b/src/tests/contention_detectors/overload_test.cpp @@ -50,8 +50,10 @@ TEST(OverloadDetectorTest, LowUtilization) { OverloadDetector overloadDetector( &mockSink, usage::getCpuUsage, - createThresholdDetectorCfg( - UTIL_THRESHOLD)); + SerenityConfig()); + + overloadDetector.setCfgUtilizationThreshold() + // Fake slave ResourceUsage source. MockSource usageSource(&overloadDetector); diff --git a/src/tests/serenity/config_test.cpp b/src/tests/serenity/config_test.cpp index c0bcf7c..82f9377 100644 --- a/src/tests/serenity/config_test.cpp +++ b/src/tests/serenity/config_test.cpp @@ -28,69 +28,101 @@ const constexpr char* FIELD_DOUBLE = "FIELD_DOUBLE"; const constexpr double_t DEFAULT_FIELD_DOUBLE = 0.345345; const constexpr double_t MODIFIED_FIELD_DOUBLE = 3.432; - -SerenityConfig loadSampleConfig() { - SerenityConfig config; - config.set(FIELD_STR, (std::string) MODIFIED_FIELD_STR); - config.set(FIELD_BOOL, MODIFIED_FIELD_BOOL); - config.set(FIELD_INT, MODIFIED_FIELD_INT); - config.set(FIELD_DOUBLE, MODIFIED_FIELD_DOUBLE); - - return config; -} +const constexpr char* FIELD_SECTION = "SECTION1"; + +class TestConfig : SerenityConfig { + public: + void loadSampleConfig() { + put(FIELD_STR, (std::string) MODIFIED_FIELD_STR); + put(FIELD_BOOL, MODIFIED_FIELD_BOOL); + put(FIELD_INT, MODIFIED_FIELD_INT); + put(FIELD_DOUBLE, MODIFIED_FIELD_DOUBLE); + } + + void loadSampleConfigWithSections() { + SerenityConfig config = getSectionOrNew(FIELD_SECTION); + config.put(FIELD_STR, (std::string) MODIFIED_FIELD_STR); + } +}; TEST(SerenityConfigTest, EmptyItemsTest) { SerenityConfig config; - EXPECT_NONE(config.item(FIELD_STR)); - EXPECT_NONE(config.item(FIELD_BOOL)); - EXPECT_NONE(config.item(FIELD_INT)); - EXPECT_NONE(config.item(FIELD_DOUBLE)); + EXPECT_NONE(config.getItem(FIELD_STR)); + EXPECT_NONE(config.getItem(FIELD_BOOL)); + EXPECT_NONE(config.getItem(FIELD_INT)); + EXPECT_NONE(config.getItem(FIELD_DOUBLE)); } TEST(SerenityConfigTest, DefaultItemsTest) { SerenityConfig config; - EXPECT_EQ(config.item(FIELD_STR, DEFAULT_FIELD_STR), + EXPECT_EQ(config.getItemOrDefault(FIELD_STR, DEFAULT_FIELD_STR), DEFAULT_FIELD_STR); - EXPECT_EQ(config.item(FIELD_BOOL, DEFAULT_FIELD_BOOL), + EXPECT_EQ(config.getItemOrDefault(FIELD_BOOL, DEFAULT_FIELD_BOOL), DEFAULT_FIELD_BOOL); - EXPECT_EQ(config.item(FIELD_INT, DEFAULT_FIELD_INT), + EXPECT_EQ(config.getItemOrDefault(FIELD_INT, DEFAULT_FIELD_INT), DEFAULT_FIELD_INT); - EXPECT_EQ(config.item(FIELD_DOUBLE, DEFAULT_FIELD_DOUBLE), + EXPECT_EQ(config.getItemOrDefault(FIELD_DOUBLE, DEFAULT_FIELD_DOUBLE), DEFAULT_FIELD_DOUBLE); } TEST(SerenityConfigTest, ModifiedItemsTest) { - SerenityConfig config; - EXPECT_EQ(config.item(FIELD_STR, DEFAULT_FIELD_STR), + TestConfig config; + EXPECT_EQ(config.getItemOrDefault(FIELD_STR, DEFAULT_FIELD_STR), DEFAULT_FIELD_STR); - EXPECT_EQ(config.item(FIELD_BOOL, DEFAULT_FIELD_BOOL), + EXPECT_EQ(config.getItemOrDefault(FIELD_BOOL, DEFAULT_FIELD_BOOL), DEFAULT_FIELD_BOOL); - EXPECT_EQ(config.item(FIELD_INT, DEFAULT_FIELD_INT), + EXPECT_EQ(config.getItemOrDefault(FIELD_INT, DEFAULT_FIELD_INT), DEFAULT_FIELD_INT); - EXPECT_EQ(config.item(FIELD_DOUBLE, DEFAULT_FIELD_DOUBLE), + EXPECT_EQ(config.getItemOrDefault(FIELD_DOUBLE, DEFAULT_FIELD_DOUBLE), DEFAULT_FIELD_DOUBLE); - config = loadSampleConfig(); + config.loadSampleConfig(); - EXPECT_EQ(config.item(FIELD_STR, DEFAULT_FIELD_STR), + EXPECT_EQ(config.getItemOrDefault(FIELD_STR, DEFAULT_FIELD_STR), MODIFIED_FIELD_STR); - EXPECT_EQ(config.item(FIELD_BOOL, DEFAULT_FIELD_BOOL), + EXPECT_EQ(config.getItemOrDefault(FIELD_BOOL, DEFAULT_FIELD_BOOL), MODIFIED_FIELD_BOOL); - EXPECT_EQ(config.item(FIELD_INT, DEFAULT_FIELD_INT), + EXPECT_EQ(config.getItemOrDefault(FIELD_INT, DEFAULT_FIELD_INT), MODIFIED_FIELD_INT); - EXPECT_EQ(config.item(FIELD_DOUBLE, DEFAULT_FIELD_DOUBLE), + EXPECT_EQ(config.getItemOrDefault(FIELD_DOUBLE, DEFAULT_FIELD_DOUBLE), MODIFIED_FIELD_DOUBLE); } TEST(SerenityConfigTest, ErrorItemsTest) { - SerenityConfig config = loadSampleConfig(); - EXPECT_ERROR(config.item(FIELD_STR)); - EXPECT_ERROR(config.item(FIELD_BOOL)); - EXPECT_ERROR(config.item(FIELD_INT)); - EXPECT_ERROR(config.item(FIELD_DOUBLE)); + TestConfig config; + config.loadSampleConfig(); + EXPECT_ERROR(config.getItem(FIELD_STR)); + EXPECT_ERROR(config.getItem(FIELD_BOOL)); + EXPECT_ERROR(config.getItem(FIELD_INT)); + EXPECT_ERROR(config.getItem(FIELD_DOUBLE)); } +TEST(SerenityConfigTest, ModifiedSectionItemsTest) { + TestConfig config; + EXPECT_EQ(config.getItemOrDefault(FIELD_STR, DEFAULT_FIELD_STR), + DEFAULT_FIELD_STR); + EXPECT_EQ(config.getItemOrDefault(FIELD_BOOL, DEFAULT_FIELD_BOOL), + DEFAULT_FIELD_BOOL); + EXPECT_EQ(config.getItemOrDefault(FIELD_INT, DEFAULT_FIELD_INT), + DEFAULT_FIELD_INT); + EXPECT_EQ(config.getItemOrDefault(FIELD_DOUBLE, DEFAULT_FIELD_DOUBLE), + DEFAULT_FIELD_DOUBLE); + + config.loadSampleConfig(); + + EXPECT_EQ(config.getItemOrDefault(FIELD_STR, DEFAULT_FIELD_STR), + MODIFIED_FIELD_STR); + EXPECT_EQ(config.getItemOrDefault(FIELD_BOOL, DEFAULT_FIELD_BOOL), + MODIFIED_FIELD_BOOL); + EXPECT_EQ(config.getItemOrDefault(FIELD_INT, DEFAULT_FIELD_INT), + MODIFIED_FIELD_INT); + EXPECT_EQ(config.getItemOrDefault(FIELD_DOUBLE, DEFAULT_FIELD_DOUBLE), + MODIFIED_FIELD_DOUBLE); +} + + + } // namespace tests } // namespace serenity } // namespace mesos From e48e3d5ecb3551345f715770c833323f754c0cbb Mon Sep 17 00:00:00 2001 From: bplotka Date: Fri, 26 Feb 2016 11:43:24 +0100 Subject: [PATCH 7/7] Big Config refactored. Added tests. Signed-off-by: bplotka --- src/contention_detectors/overload.cpp | 2 + src/contention_detectors/overload.hpp | 19 +- .../signal_analyzers/drop.cpp | 21 +- .../signal_analyzers/drop.hpp | 106 ++++++----- src/contention_detectors/signal_based.hpp | 4 +- src/filters/ema.hpp | 8 +- src/filters/ignore_new_executors.hpp | 5 +- src/filters/too_low_usage.cpp | 7 +- src/filters/too_low_usage.hpp | 21 +- src/filters/utilization_threshold.hpp | 7 +- .../serenity_controller_module.cpp | 21 +- src/observers/qos_correction.cpp | 2 +- src/observers/qos_correction.hpp | 12 +- src/observers/slack_resource.hpp | 15 +- src/observers/strategies/cache_occupancy.hpp | 6 +- src/observers/strategies/cpu_contention.cpp | 4 +- src/observers/strategies/cpu_contention.hpp | 21 +- src/observers/strategies/seniority.cpp | 1 + src/observers/strategies/seniority.hpp | 14 +- src/pipeline/estimator_pipeline.hpp | 8 +- src/pipeline/qos_pipeline.hpp | 68 +++---- src/serenity/config.hpp | 180 ++++++++++-------- src/serenity/default_vars.hpp | 79 -------- src/serenity/resource_helper.cpp | 2 +- src/tests/common/config_helper.hpp | 42 ---- .../contention_detectors/overload_test.cpp | 15 +- .../signal_analyzers/drop_test.cpp | 118 ++++++------ .../qos_controller/qos_controller_test.cpp | 2 +- .../strategies/seniority_strategy_test.cpp | 4 +- src/tests/serenity/config_test.cpp | 134 ++++++++----- 30 files changed, 482 insertions(+), 466 deletions(-) delete mode 100644 src/serenity/default_vars.hpp delete mode 100644 src/tests/common/config_helper.hpp diff --git a/src/contention_detectors/overload.cpp b/src/contention_detectors/overload.cpp index 178ca03..902f803 100644 --- a/src/contention_detectors/overload.cpp +++ b/src/contention_detectors/overload.cpp @@ -10,6 +10,8 @@ namespace mesos { namespace serenity { +const constexpr char* OverloadDetector::UTILIZATION_THRESHOLD_KEY; + void OverloadDetector::allProductsReady() { Contentions product; ResourceUsage usage = getConsumable().get(); diff --git a/src/contention_detectors/overload.hpp b/src/contention_detectors/overload.hpp index 1992e8e..6c2d118 100644 --- a/src/contention_detectors/overload.hpp +++ b/src/contention_detectors/overload.hpp @@ -32,26 +32,29 @@ class OverloadDetector : OverloadDetector( Consumer* _consumer, const lambda::function& _cpuUsageGetFunction, - const SerenityConfig& _conf, + const Config& _conf, const Tag& _tag = Tag(QOS_CONTROLLER, NAME)) : tag(_tag), cpuUsageGetFunction(_cpuUsageGetFunction), Producer(_consumer) { // Parse config values. - setCfgUtilizationThreshold( - _conf.getItemOrDefault( - detector::THRESHOLD, - detector::DEFAULT_UTILIZATION_THRESHOLD)); + setUtilizationThreshold( + _conf.getValue(UTILIZATION_THRESHOLD_KEY)); } ~OverloadDetector() {} static const constexpr char* NAME = "OverloadDetector"; - void setCfgUtilizationThreshold(double_t cfgUtilizationThreshold) { - OverloadDetector::cfgUtilizationThreshold = cfgUtilizationThreshold; + void setUtilizationThreshold(const Result& value) { + cfgUtilizationThreshold = + ConfigValidator(value, UTILIZATION_THRESHOLD_KEY) + .validateValueIsPositive() + .getOrElse(DEFAULT_UTILIZATION_THRESHOLD); } + static const constexpr char* UTILIZATION_THRESHOLD_KEY = "THRESHOLD"; + protected: void allProductsReady() override; bool hasRequiredFields(const ResourceUsage_Executor& inExec); @@ -61,6 +64,8 @@ class OverloadDetector : // cfg parameters. double_t cfgUtilizationThreshold; + + static const constexpr double_t DEFAULT_UTILIZATION_THRESHOLD = 0.72; }; } // namespace serenity diff --git a/src/contention_detectors/signal_analyzers/drop.cpp b/src/contention_detectors/signal_analyzers/drop.cpp index 75bdb6e..bcc7047 100644 --- a/src/contention_detectors/signal_analyzers/drop.cpp +++ b/src/contention_detectors/signal_analyzers/drop.cpp @@ -11,6 +11,15 @@ namespace mesos { namespace serenity { + +const constexpr char* SignalDropAnalyzer::WINDOW_SIZE_KEY; +const constexpr char* SignalDropAnalyzer::FRACTIONAL_THRESHOLD_KEY; +const constexpr char* SignalDropAnalyzer::SEVERITY_FRACTION_KEY; +const constexpr char* SignalDropAnalyzer::NEAR_FRACTION_KEY; +const constexpr char* SignalDropAnalyzer::MAX_CHECKPOINTS_KEY; +const constexpr char* SignalDropAnalyzer::QUORUM_FRACTION_KEY; +const constexpr double_t SignalDropAnalyzer::START_VALUE_DEFAULT; + void SignalDropAnalyzer::shiftBasePoints() { for (std::list::iterator& basePoint : this->basePoints) { basePoint++; @@ -32,14 +41,14 @@ void SignalDropAnalyzer::recalculateParams() { } // Make sure it does not exceed MAX_CHECKPOINTS option. - checkpoints = min(checkpoints, cfgMaxCheckpoints); + checkpoints = std::min((int64_t)checkpoints, cfgMaxCheckpoints); // Get the Quorum number from QUORUM fraction parameter. quorumNum = cfgQuroumFraction * checkpoints; if (quorumNum == 0 || quorumNum > checkpoints) { SERENITY_LOG(WARNING) << "Bad value for Quorum parameter. Creating 100%" << " quorum."; - quorumNum = checkpoints; + quorumNum = checkpoints; } std::stringstream checkpointLog; @@ -49,7 +58,7 @@ void SignalDropAnalyzer::recalculateParams() { // from the end of window. uint64_t choosenNum = pow(2, (--checkpoints)); for (uint64_t i = cfgWindowSize; i > 0 ; i--) { - window.push_back(detector::DEFAULT_START_VALUE); + window.push_back(START_VALUE_DEFAULT); if (choosenNum == i) { checkpointLog << "T-" << choosenNum << " "; @@ -65,6 +74,12 @@ void SignalDropAnalyzer::recalculateParams() { Result SignalDropAnalyzer::processSample(double_t in) { + if (paramsChanged) { + recalculateParams(); + paramsChanged = false; + } + + // Fill window. if (in < 0.1) in = 0.1; diff --git a/src/contention_detectors/signal_analyzers/drop.hpp b/src/contention_detectors/signal_analyzers/drop.hpp index 775be6f..988f0cd 100644 --- a/src/contention_detectors/signal_analyzers/drop.hpp +++ b/src/contention_detectors/signal_analyzers/drop.hpp @@ -52,36 +52,18 @@ class SignalDropAnalyzer : public SignalAnalyzer { public: explicit SignalDropAnalyzer( const Tag& _tag, - const SerenityConfig& _config) + const Config& _config) : SignalAnalyzer(_tag), valueBeforeDrop(None()), quorumNum(0) { - setWindowsSizeAndMaxCheckpoints( - _config.getItemAndSetDefault( - SignalDropAnalyzer::WINDOW_SIZE_KEY, - SignalDropAnalyzer::WINDOW_SIZE_DEFAULT), - _config.getItemAndSetDefault( - SignalDropAnalyzer::MAX_CHECKPOINTS_KEY, - SignalDropAnalyzer::MAX_CHECKPOINTS_DEFAULT)); - - setQuroumFraction(_config.getItemAndSetDefault( - SignalDropAnalyzer::QUORUM_FRACTION_KEY, - SignalDropAnalyzer::QUORUM_FRACTION_DEFAULT)); - - setFractionalThreshold(_config.getItemAndSetDefault( - SignalDropAnalyzer::FRACTIONAL_THRESHOLD_KEY, - SignalDropAnalyzer::FRACTIONAL_THRESHOLD_DEFAULT)); - - setNearFraction(_config.getItemAndSetDefault( - SignalDropAnalyzer::NEAR_FRACTION_KEY, - SignalDropAnalyzer::NEAR_FRACTION_DEFAULT)); - - setSeverityFraction(_config.getItemAndSetDefault( - SignalDropAnalyzer::SEVERITY_FRACTION_KEY, - SignalDropAnalyzer::SEVERITY_FRACTION_DEFAULT)); - - this->recalculateParams(); + _config.getValue(WINDOW_SIZE_KEY), + _config.getValue(MAX_CHECKPOINTS_KEY)); + setQuroumFraction(_config.getValue(QUORUM_FRACTION_KEY)); + setFractionalThreshold( + _config.getValue(FRACTIONAL_THRESHOLD_KEY)); + setNearFraction(_config.getValue(NEAR_FRACTION_KEY)); + setSeverityFraction(_config.getValue(SEVERITY_FRACTION_KEY)); } Result _processSample(double_t in); @@ -95,17 +77,6 @@ class SignalDropAnalyzer : public SignalAnalyzer { */ void shiftBasePoints(); - static const constexpr char* WINDOW_SIZE_KEY = "WINDOW_SIZE"; - static const constexpr char* FRACTIONAL_THRESHOLD_KEY = - "FRACTIONAL_THRESHOLD"; - static const constexpr char* SEVERITY_FRACTION_KEY = "SEVERITY_FRACTION"; - static const constexpr char* NEAR_FRACTION_KEY = "NEAR_FRACTION"; - static const constexpr char* MAX_CHECKPOINTS_KEY = "MAX_CHECKPOINTS"; - static const constexpr char* QUORUM_FRACTION_KEY = "QUORUM_FRACTION"; - -protected: - void recalculateParams(); - //! WindowSize: How far in the past we look. //! MaxCheckpoints: Maximum number of checkpoints we will have in our //! assurance detector. @@ -113,38 +84,69 @@ class SignalDropAnalyzer : public SignalAnalyzer { //! to in the past when detecting drop or not. //! It needs to be 0 < and < WINDOW_SIZE. void setWindowsSizeAndMaxCheckpoints( - SerenityItem _cfgWindowSize, - SerenityItem _cfgMaxCheckpoints) { - cfgWindowSize = _cfgWindowSize.validateValueIsPositive().getValueOrDefault(); - cfgMaxCheckpoints = _cfgMaxCheckpoints + const Result& _cfgWindowSize, + const Result& _cfgMaxCheckpoints) { + cfgWindowSize = ConfigValidator(_cfgWindowSize, WINDOW_SIZE_KEY) .validateValueIsPositive() - .validateValueIsBelow(_cfgWindowSize) - .getValueOrDefault(); + .getOrElse(WINDOW_SIZE_DEFAULT); + + cfgMaxCheckpoints = + ConfigValidator(_cfgMaxCheckpoints, MAX_CHECKPOINTS_KEY) + .validateValueIsPositive() + .validateValueIsBelow(cfgWindowSize, WINDOW_SIZE_KEY) + .getOrElse(MAX_CHECKPOINTS_DEFAULT); + paramsChanged = true; } //! Fraction of checkpoints' votes needed to make a Drop contention. - void setQuroumFraction(SerenityItem item) { - cfgQuroumFraction = item.validateValueIsPositive().getValueOrDefault(); + void setQuroumFraction(const Result& value) { + cfgQuroumFraction = ConfigValidator(value, QUORUM_FRACTION_KEY) + .validateValueIsPositive() + .getOrElse(QUORUM_FRACTION_DEFAULT); + paramsChanged = true; } //! Defines how much (relatively to base point) value must drop to trigger //! contention. - void setFractionalThreshold(SerenityItem item) { - cfgFractionalThreshold = item.validateValueIsPositive().getValueOrDefault(); + void setFractionalThreshold(const Result& value) { + cfgFractionalThreshold = + ConfigValidator(value, FRACTIONAL_THRESHOLD_KEY) + .validateValueIsPositive() + .getOrElse(FRACTIONAL_THRESHOLD_DEFAULT); + paramsChanged = true; } //! You can adjust how big severity is created for a defined drop. //! if -1 then unknown severity will be reported. - void setSeverityFraction(SerenityItem item) { - cfgSeverityFraction = item.getValueOrDefault(); + void setSeverityFraction(const Result& value) { + cfgSeverityFraction = + ConfigValidator(value, SEVERITY_FRACTION_KEY) + .getOrElse(SEVERITY_FRACTION_DEFAULT); + paramsChanged = true; } //! Tolerance fraction of threshold if signal is accepted as returned to //! previous state after drop. - void setNearFraction(SerenityItem item) { - cfgNearFraction = item.validateValueIsPositive().getValueOrDefault(); + void setNearFraction(const Result& value) { + cfgNearFraction = + ConfigValidator(value, NEAR_FRACTION_KEY) + .validateValueIsPositive() + .getOrElse(NEAR_FRACTION_DEFAULT); + paramsChanged = true; } + static const constexpr char* NAME = "SignalDropAnalyzer"; + static const constexpr char* WINDOW_SIZE_KEY = "WINDOW_SIZE"; + static const constexpr char* FRACTIONAL_THRESHOLD_KEY = + "FRACTIONAL_THRESHOLD"; + static const constexpr char* SEVERITY_FRACTION_KEY = "SEVERITY_FRACTION"; + static const constexpr char* NEAR_FRACTION_KEY = "NEAR_FRACTION"; + static const constexpr char* MAX_CHECKPOINTS_KEY = "MAX_CHECKPOINTS"; + static const constexpr char* QUORUM_FRACTION_KEY = "QUORUM_FRACTION"; + + protected: + void recalculateParams(); + std::list window; std::list::iterator> basePoints; @@ -153,6 +155,7 @@ class SignalDropAnalyzer : public SignalAnalyzer { uint64_t dropVotes; uint64_t quorumNum; + bool paramsChanged = true; // Cfg parameters. int64_t cfgWindowSize; @@ -169,6 +172,7 @@ class SignalDropAnalyzer : public SignalAnalyzer { static constexpr double_t NEAR_FRACTION_DEFAULT = 0.1; static constexpr int64_t MAX_CHECKPOINTS_DEFAULT = 3; static constexpr double_t QUORUM_FRACTION_DEFAULT = 0.7; + static constexpr double_t START_VALUE_DEFAULT = 0.00001; }; diff --git a/src/contention_detectors/signal_based.hpp b/src/contention_detectors/signal_based.hpp index 412330c..cc5aa4a 100644 --- a/src/contention_detectors/signal_based.hpp +++ b/src/contention_detectors/signal_based.hpp @@ -42,7 +42,7 @@ class SignalBasedDetector : SignalBasedDetector( Consumer* _consumer, const lambda::function& _getValue, - SerenityConfig _detectorConf, + const Config& _detectorConf, const Tag& _tag = Tag(QOS_CONTROLLER, "SignalBasedDetector"), const Contention_Type _contentionType = Contention_Type_IPC) : tag(_tag), @@ -65,7 +65,7 @@ class SignalBasedDetector : // Detections. ExecutorMap> detectors; - SerenityConfig detectorConf; + const Config detectorConf; }; } // namespace serenity diff --git a/src/filters/ema.hpp b/src/filters/ema.hpp index d3e294c..d406aae 100644 --- a/src/filters/ema.hpp +++ b/src/filters/ema.hpp @@ -10,7 +10,6 @@ #include "messages/serenity.hpp" #include "serenity/data_utils.hpp" -#include "serenity/default_vars.hpp" #include "serenity/executor_map.hpp" #include "serenity/executor_set.hpp" #include "serenity/serenity.hpp" @@ -36,7 +35,7 @@ class ExponentialMovingAverage { public: ExponentialMovingAverage( EMASeriesType _seriesType = EMA_REGULAR_SERIES, - double_t _alpha = ema::DEFAULT_ALPHA) + double_t _alpha = ALPHA_DEFAULT) : alpha(_alpha), seriesType(_seriesType), uninitialized(true) {} @@ -55,6 +54,7 @@ class ExponentialMovingAverage { double_t calculateEMA(double_t sample, double_t sampleTimestamp); private: + static const constexpr double_t ALPHA_DEFAULT = 0.2; //! Constant describing how the window weights decrease over time. //! It controls how long the moving average period is. //! The smaller alpha becomes, the longer your moving average is. @@ -103,7 +103,7 @@ class EMAFilter : Consumer* _consumer, const lambda::function& _valueGetFunction, const lambda::function& _valueSetFunction, - double_t _alpha = ema::DEFAULT_ALPHA, + double_t _alpha = ALPHA_DEFAULT, const Tag& _tag = Tag(UNDEFINED, "emaFilter")) : tag(_tag), Producer(_consumer), emaSamples(new ExecutorMap()), @@ -121,6 +121,8 @@ class EMAFilter : const lambda::function valueGetFunction; const lambda::function valueSetFunction; std::unique_ptr> emaSamples; + + static const constexpr double_t ALPHA_DEFAULT = 0.2; }; } // namespace serenity diff --git a/src/filters/ignore_new_executors.hpp b/src/filters/ignore_new_executors.hpp index eb0d150..34285ed 100644 --- a/src/filters/ignore_new_executors.hpp +++ b/src/filters/ignore_new_executors.hpp @@ -9,7 +9,6 @@ #include "messages/serenity.hpp" -#include "serenity/default_vars.hpp" #include "serenity/executor_map.hpp" #include "serenity/serenity.hpp" @@ -32,7 +31,7 @@ class IgnoreNewExecutorsFilter : public Consumer, public: explicit IgnoreNewExecutorsFilter( Consumer* _consumer = nullptr, - uint32_t _thresholdSeconds = new_executor::DEFAULT_THRESHOLD_SEC) : + uint32_t _thresholdSeconds = THRESHOLD_SEC_DEFAULT) : Producer(_consumer), threshold(_thresholdSeconds), executorTimestamps(new ExecutorMap) {} @@ -61,6 +60,8 @@ class IgnoreNewExecutorsFilter : public Consumer, static constexpr const char* name = "[SerenityEstimator] IgnoreNewExecutorsFilter: "; + + static constexpr uint32_t THRESHOLD_SEC_DEFAULT = 5 * 60; // !< Five minutes. }; } // namespace serenity diff --git a/src/filters/too_low_usage.cpp b/src/filters/too_low_usage.cpp index 66154c3..48cae01 100644 --- a/src/filters/too_low_usage.cpp +++ b/src/filters/too_low_usage.cpp @@ -18,6 +18,9 @@ using std::map; using std::pair; using std::string; + +const constexpr char* TooLowUsageFilter::MINIMAL_CPU_USAGE_KEY; + TooLowUsageFilter::~TooLowUsageFilter() {} @@ -58,11 +61,11 @@ Try TooLowUsageFilter::consume(const ResourceUsage& in) { continue; } - if (cpuUsage.get() <= this->cfgMinimalCpuUsage) { + if (cpuUsage.get() <= cfgMinimalCpuUsage) { // Exclude executor. SERENITY_LOG(INFO) << "Filtering out PR exec: " << executor_id << " because of its CPU Usage: " << cpuUsage.get() << " [Min: " - << this->cfgMinimalCpuUsage << "]"; + << cfgMinimalCpuUsage << "]"; continue; } } diff --git a/src/filters/too_low_usage.hpp b/src/filters/too_low_usage.hpp index bf44b32..824e178 100644 --- a/src/filters/too_low_usage.hpp +++ b/src/filters/too_low_usage.hpp @@ -26,28 +26,35 @@ class TooLowUsageFilter : explicit TooLowUsageFilter( Consumer* _consumer, - const SerenityConfig& _conf, + const Config& _conf, const Tag& _tag = Tag(QOS_CONTROLLER, NAME)) : Producer(_consumer), tag(_tag) { - setCfgMinimalCpuUsage(_conf.getItemOrDefault( - too_low_usage::MINIMAL_CPU_USAGE, - too_low_usage::DEFAULT_MINIMAL_CPU_USAGE)); + setMinimalCpuUsage(_conf.getValue(MINIMAL_CPU_USAGE_KEY)); } ~TooLowUsageFilter(); - static const constexpr char* NAME = "TooLowUsageFilter"; + Try consume(const ResourceUsage& in); - void setCfgMinimalCpuUsage(double_t cfgMinimalCpuUsage) { - TooLowUsageFilter::cfgMinimalCpuUsage = cfgMinimalCpuUsage; + void setMinimalCpuUsage(Result cfgMinimalCpuUsage) { + cfgMinimalCpuUsage = + ConfigValidator(cfgMinimalCpuUsage, MINIMAL_CPU_USAGE_KEY) + .validateValueIsPositive() + .getOrElse(MINIMAL_CPU_USAGE_DEFAULT); } + static const constexpr char* NAME = "TooLowUsageFilter"; + + static const constexpr char* MINIMAL_CPU_USAGE_KEY = "MINIMAL_CPU_USAGE"; + protected: const Tag tag; double_t cfgMinimalCpuUsage; + + static constexpr double_t MINIMAL_CPU_USAGE_DEFAULT = 0.25; // !< per sec. }; } // namespace serenity diff --git a/src/filters/utilization_threshold.hpp b/src/filters/utilization_threshold.hpp index df0ae5e..b7097a9 100644 --- a/src/filters/utilization_threshold.hpp +++ b/src/filters/utilization_threshold.hpp @@ -4,7 +4,6 @@ #include #include -#include "serenity/default_vars.hpp" #include "serenity/executor_set.hpp" #include "serenity/serenity.hpp" @@ -27,7 +26,7 @@ class UtilizationThresholdFilter : public Consumer, public Producer { public: UtilizationThresholdFilter( - double_t _utilizationThreshold = utilization::DEFAULT_THRESHOLD, + double_t _utilizationThreshold = THRESHOLD_DEFAULT, const Tag& _tag = Tag(UNDEFINED, "utilizationFilter")) : tag(_tag), utilizationThreshold(_utilizationThreshold), @@ -35,7 +34,7 @@ class UtilizationThresholdFilter : UtilizationThresholdFilter( Consumer* _consumer, - double_t _utilizationThreshold = utilization::DEFAULT_THRESHOLD, + double_t _utilizationThreshold = THRESHOLD_DEFAULT, const Tag& _tag = Tag(UNDEFINED, "utilizationFilter")) : tag(_tag), Producer(_consumer), utilizationThreshold(_utilizationThreshold), @@ -56,6 +55,8 @@ class UtilizationThresholdFilter : const std::string UTILIZATION_THRESHOLD_FILTER_WARNING = "Filter is not" \ "able to calculate total cpu usage and will base on allocated " \ " resources to cut off oversubscription if needed."; + + static constexpr double_t THRESHOLD_DEFAULT = 0.95; }; } // namespace serenity diff --git a/src/mesos_modules/qos_controller/serenity_controller_module.cpp b/src/mesos_modules/qos_controller/serenity_controller_module.cpp index fceb134..7bc05ea 100644 --- a/src/mesos_modules/qos_controller/serenity_controller_module.cpp +++ b/src/mesos_modules/qos_controller/serenity_controller_module.cpp @@ -17,15 +17,11 @@ // TODO(nnielsen): Should be explicit using-directives. using namespace mesos; // NOLINT(build/namespaces) -using namespace mesos::serenity::ema; // NOLINT(build/namespaces) -using namespace mesos::serenity::strategy; // NOLINT(build/namespaces) -using namespace mesos::serenity::detector; // NOLINT(build/namespaces) -using namespace mesos::serenity::too_low_usage; // NOLINT(build/namespaces) -using namespace mesos::serenity::qos_pipeline; // NOLINT(build/namespaces) using mesos::serenity::CpuContentionStrategy; using mesos::serenity::CpuQoSPipeline; -using mesos::serenity::SerenityConfig; +using mesos::serenity::Config; +using mesos::serenity::ConfigValidator; using mesos::serenity::SerenityController; using mesos::serenity::SeniorityStrategy; using mesos::serenity::SignalBasedDetector; @@ -34,17 +30,22 @@ using mesos::serenity::QoSControllerPipeline; using mesos::slave::QoSController; +const constexpr char* ON_EMPTY_CORRECTION_INTERVAL_KEY = + "ON_EMPTY_CORRECTION_INTERVAL"; + +const constexpr double_t ON_EMPTY_CORRECTION_INTERVAL_DEFAULT = 2; // IPC QoS pipeline. static QoSController* createSerenityController( const Parameters& parameters) { LOG(INFO) << "Loading Serenity QoS Controller module"; // TODO(bplotka): Fetch configuration from parameters or conf file. + Config conf; - SerenityConfig conf; - double onEmptyCorrectionInterval = - conf.getItemOrDefault(ON_EMPTY_CORRECTION_INTERVAL, - DEFAULT_ON_EMPTY_CORRECTION_INTERVAL); + double_t onEmptyCorrectionInterval = + ConfigValidator( + conf.getValue(ON_EMPTY_CORRECTION_INTERVAL_KEY)) + .getOrElse(ON_EMPTY_CORRECTION_INTERVAL_DEFAULT); // Use static constructor of QoSController. Try result = diff --git a/src/observers/qos_correction.cpp b/src/observers/qos_correction.cpp index 8bc1d98..c946e5e 100644 --- a/src/observers/qos_correction.cpp +++ b/src/observers/qos_correction.cpp @@ -57,7 +57,7 @@ void QoSCorrectionObserver::allProductsReady() { // Strategy has pointed aggressors, so don't pass // current contentions to next QoS Controller. - iterationCooldownCounter = this->cooldownIterations; + iterationCooldownCounter = cfgCooldownIterations; produceResults(corrections.get(), Contentions()); } diff --git a/src/observers/qos_correction.hpp b/src/observers/qos_correction.hpp index 7b43510..4ddb4f3 100644 --- a/src/observers/qos_correction.hpp +++ b/src/observers/qos_correction.hpp @@ -51,13 +51,13 @@ class QoSCorrectionObserver : public Consumer, Consumer* _consumer, ExecutorAgeFilter* _ageFilter = new ExecutorAgeFilter(), RevocationStrategy* _revStrategy = - new SeniorityStrategy(SerenityConfig()), - uint32_t _cooldownIterations = strategy::DEFAULT_CONTENTION_COOLDOWN, + new SeniorityStrategy(Config()), + int64_t _cooldownIterations = CONTENTION_COOLDOWN_DEFAULT, const Tag& _tag = Tag(QOS_CONTROLLER, NAME)) : Producer(_consumer), revocationStrategy(_revStrategy), executorAgeFilter(_ageFilter), - cooldownIterations(_cooldownIterations), + cfgCooldownIterations(_cooldownIterations), tag(_tag) {} ~QoSCorrectionObserver(); @@ -69,6 +69,8 @@ class QoSCorrectionObserver : public Consumer, static constexpr const char* NAME = "QoSCorrectionObserver"; + static const constexpr char* CONTENTION_COOLDOWN_KEY = "CONTENTION_COOLDOWN"; + protected: void allProductsReady() override; @@ -111,7 +113,9 @@ class QoSCorrectionObserver : public Consumer, /** * Default iterationCooldownCounter start value. */ - uint64_t cooldownIterations; + int64_t cfgCooldownIterations; + + static constexpr int64_t CONTENTION_COOLDOWN_DEFAULT = 10; }; } // namespace serenity diff --git a/src/observers/slack_resource.hpp b/src/observers/slack_resource.hpp index 3883971..c9abad2 100644 --- a/src/observers/slack_resource.hpp +++ b/src/observers/slack_resource.hpp @@ -11,7 +11,6 @@ #include "stout/result.hpp" -#include "serenity/default_vars.hpp" #include "serenity/executor_set.hpp" #include "serenity/serenity.hpp" @@ -39,16 +38,16 @@ class SlackResourceObserver : public Consumer, public: explicit SlackResourceObserver( double_t _maxOversubscriptionFraction = - slack_observer::DEFAULT_MAX_OVERSUBSCRIPTION_FRACTION) - : previousSamples(new ExecutorSet()), - maxOversubscriptionFraction(_maxOversubscriptionFraction), - default_role(getDefaultRole()) {} + MAX_OVERSUBSCRIPTION_FRACTION_DEFAULT) + : previousSamples(new ExecutorSet()), + maxOversubscriptionFraction(_maxOversubscriptionFraction), + default_role(getDefaultRole()) {} SlackResourceObserver( Consumer* _consumer, double_t _maxOversubscriptionFraction = - slack_observer::DEFAULT_MAX_OVERSUBSCRIPTION_FRACTION) : - Producer(_consumer), + MAX_OVERSUBSCRIPTION_FRACTION_DEFAULT) + : Producer(_consumer), maxOversubscriptionFraction(_maxOversubscriptionFraction), previousSamples(new ExecutorSet()), default_role(getDefaultRole()) {} @@ -76,6 +75,8 @@ class SlackResourceObserver : public Consumer, SlackResourceObserver(const SlackResourceObserver& other) : default_role(getDefaultRole()) {} std::string default_role; + + static const constexpr double_t MAX_OVERSUBSCRIPTION_FRACTION_DEFAULT = 0.8; }; } // namespace serenity diff --git a/src/observers/strategies/cache_occupancy.hpp b/src/observers/strategies/cache_occupancy.hpp index 1fe7e58..777338b 100644 --- a/src/observers/strategies/cache_occupancy.hpp +++ b/src/observers/strategies/cache_occupancy.hpp @@ -28,7 +28,7 @@ class CacheOccupancyStrategy : public RevocationStrategy { init(); } - explicit CacheOccupancyStrategy(const SerenityConfig& _config) + explicit CacheOccupancyStrategy(const Config& _config) : RevocationStrategy(Tag(QOS_CONTROLLER, NAME)) { init(); } @@ -42,7 +42,7 @@ class CacheOccupancyStrategy : public RevocationStrategy { protected: void init() { - minimalCacheOccupancy = DEFAULT_MINIMAL_CACHE_OCCUPANCY; + minimalCacheOccupancy = MINIMAL_CACHE_OCCUPANCY_DEFAULT; } std::vector getCmtEnabledExecutors( @@ -56,7 +56,7 @@ class CacheOccupancyStrategy : public RevocationStrategy { const double_t meanCacheOccupancy) const; //!< Minimal cache occupancy for executor to be revoked. - static constexpr uint64_t DEFAULT_MINIMAL_CACHE_OCCUPANCY = 1000000; // 1M + static constexpr uint64_t MINIMAL_CACHE_OCCUPANCY_DEFAULT = 1000000; // 1M uint64_t minimalCacheOccupancy; }; diff --git a/src/observers/strategies/cpu_contention.cpp b/src/observers/strategies/cpu_contention.cpp index caa6674..3d607e4 100644 --- a/src/observers/strategies/cpu_contention.cpp +++ b/src/observers/strategies/cpu_contention.cpp @@ -16,6 +16,8 @@ namespace serenity { using std::list; using std::pair; +const constexpr char* CpuContentionStrategy::DEFAULT_CPU_SEVERITY_KEY; + Try CpuContentionStrategy::decide( ExecutorAgeFilter* ageFilter, const Contentions& currentContentions, @@ -43,7 +45,7 @@ Try CpuContentionStrategy::decide( if (contention.has_severity()) { cpuToRecover = std::max(contention.severity(), cpuToRecover); } else { - cpuToRecover = std::max(this->defaultSeverity, cpuToRecover); + cpuToRecover = std::max(cfgDefaultSeverity, cpuToRecover); } } diff --git a/src/observers/strategies/cpu_contention.hpp b/src/observers/strategies/cpu_contention.hpp index 2ed4a4b..789e823 100644 --- a/src/observers/strategies/cpu_contention.hpp +++ b/src/observers/strategies/cpu_contention.hpp @@ -23,30 +23,35 @@ namespace serenity { class CpuContentionStrategy : public RevocationStrategy { public: explicit CpuContentionStrategy( - const SerenityConfig& _config, + const Config& _config, const lambda::function& _cpuUsageGetFunction) : RevocationStrategy(Tag(QOS_CONTROLLER, "CpuContentionStrategy")), getCpuUsage(_cpuUsageGetFunction) { - setDefaultSeverity(_config.getItemOrDefault( - strategy::DEFAULT_CPU_SEVERITY, - strategy::DEFAULT_DEFAULT_CPU_SEVERITY)); + setDefaultSeverity(_config.getValue(DEFAULT_CPU_SEVERITY_KEY)); } Try decide(ExecutorAgeFilter* ageFilter, const Contentions& currentContentions, const ResourceUsage& currentUsage); + void setDefaultSeverity(const Result value) { + cfgDefaultSeverity = + ConfigValidator(value, DEFAULT_CPU_SEVERITY_KEY) + .getOrElse(CPU_SEVERITY_DEFAULT); + } + static const constexpr char* NAME = "CpuContentionStrategy"; - void setDefaultSeverity(double_t defaultSeverity) { - CpuContentionStrategy::defaultSeverity = defaultSeverity; - } + static const constexpr char* DEFAULT_CPU_SEVERITY_KEY = + "DEFAULT_CPU_SEVERITY"; private: const lambda::function getCpuUsage; // cfg parameters. - double_t defaultSeverity; + double_t cfgDefaultSeverity; + + static constexpr double_t CPU_SEVERITY_DEFAULT = 1.0; }; } // namespace serenity diff --git a/src/observers/strategies/seniority.cpp b/src/observers/strategies/seniority.cpp index 06c21a7..cc8e3a8 100644 --- a/src/observers/strategies/seniority.cpp +++ b/src/observers/strategies/seniority.cpp @@ -15,6 +15,7 @@ using std::list; using std::pair; +const constexpr char* SeniorityStrategy::STARTING_SEVERITY_KEY; Try SeniorityStrategy::decide( ExecutorAgeFilter* ageFilter, diff --git a/src/observers/strategies/seniority.hpp b/src/observers/strategies/seniority.hpp index f204323..0abd563 100644 --- a/src/observers/strategies/seniority.hpp +++ b/src/observers/strategies/seniority.hpp @@ -24,23 +24,27 @@ class SeniorityStrategy : public RevocationStrategy { * TODO(skonefal): SerenityConfig should have const methods inside. * Currently, it cannot be passed as const. */ - explicit SeniorityStrategy(SerenityConfig _config) + explicit SeniorityStrategy(const Config& _config) : RevocationStrategy(Tag(QOS_CONTROLLER, NAME)) { - severity = _config.getItemOrDefault(STARTING_SEVERITY_KEY, - DEFAULT_SEVERITY); + setSeverity(_config.getValue(STARTING_SEVERITY_KEY)); } Try decide(ExecutorAgeFilter*, const Contentions&, const ResourceUsage&); + void setSeverity(Result value) { + severity = ConfigValidator(value, STARTING_SEVERITY_KEY) + .getOrElse(SEVERITY_DEFAULT); + } + static const constexpr char* STARTING_SEVERITY_KEY = "STARTING_SEVERITY"; static const constexpr char* NAME = "SeniorityStrategy"; private: - static const constexpr double_t DEFAULT_SEVERITY = 0.1; - double_t severity; + + static const constexpr double_t SEVERITY_DEFAULT = 0.1; }; } // namespace serenity diff --git a/src/pipeline/estimator_pipeline.hpp b/src/pipeline/estimator_pipeline.hpp index b11c994..b2c922c 100644 --- a/src/pipeline/estimator_pipeline.hpp +++ b/src/pipeline/estimator_pipeline.hpp @@ -13,7 +13,6 @@ #include "pipeline/pipeline.hpp" -#include "serenity/default_vars.hpp" #include "serenity/serenity.hpp" #include "time_series_export/slack_ts_export.hpp" @@ -56,8 +55,8 @@ using ResourceEstimatorPipeline = Pipeline; class CpuEstimatorPipeline : public ResourceEstimatorPipeline { public: explicit CpuEstimatorPipeline( - double_t _newExecutorsThreshold = new_executor::DEFAULT_THRESHOLD_SEC, - double_t _utilizationThreshold = utilization::DEFAULT_THRESHOLD, + double_t _newExecutorsThreshold = THRESHOLD_SEC_DEFAULT, + double_t _utilizationThreshold = THRESHOLD_DEFAULT, bool _visualisation = false, bool _valveOpened = true) : // Time series exporters. @@ -101,6 +100,9 @@ class CpuEstimatorPipeline : public ResourceEstimatorPipeline { PrExecutorPassFilter prExecutorPassFilter; UtilizationThresholdFilter utilizationFilter; ValveFilter valveFilter; + + static constexpr double_t THRESHOLD_DEFAULT = 0.95; + static constexpr uint32_t THRESHOLD_SEC_DEFAULT = 5 * 60; // !< Five minutes. }; } // namespace serenity diff --git a/src/pipeline/qos_pipeline.hpp b/src/pipeline/qos_pipeline.hpp index 6c93321..d3d62da 100644 --- a/src/pipeline/qos_pipeline.hpp +++ b/src/pipeline/qos_pipeline.hpp @@ -31,28 +31,6 @@ namespace mesos { namespace serenity { -using namespace qos_pipeline; // NOLINT(build/namespaces) - -class QoSPipelineConfig : public SerenityConfig { - public: - QoSPipelineConfig() {} - - explicit QoSPipelineConfig(const SerenityConfig& customCfg) { - this->initDefaults(); - this->applyConfig(customCfg); - } - - void initDefaults() { - // Used sections: QoSCorrectionObserver, AssuranceDetector, - // UtilizationDetector - // TODO(bplotka): Move EMA conf to separate section. - this->items[ema::ALPHA] = ema::DEFAULT_ALPHA; - this->items[VALVE_OPENED] = DEFAULT_VALVE_OPENED; - this->items[ENABLED_VISUALISATION] = DEFAULT_ENABLED_VISUALISATION; - } -}; - - using QoSControllerPipeline = Pipeline; @@ -102,7 +80,7 @@ using QoSControllerPipeline = Pipeline; */ class CpuQoSPipeline : public QoSControllerPipeline { public: - explicit CpuQoSPipeline(const SerenityConfig& _conf) + explicit CpuQoSPipeline(const Config& _conf) : conf(_conf), // NOTE(bplotka): age Filter should initialized first before passing // to the qosCorrectionObserver. @@ -121,20 +99,22 @@ class CpuQoSPipeline : public QoSControllerPipeline { &correctionMerger, &ageFilter, new CacheOccupancyStrategy(), - strategy::DEFAULT_CONTENTION_COOLDOWN, + ConfigValidator(conf.getValue( + QoSCorrectionObserver::CONTENTION_COOLDOWN_KEY)) + .getOrElse(CONTENTION_COOLDOWN_DEFAULT), Tag(QOS_CONTROLLER, CacheOccupancyStrategy::NAME)), ipcDropDetector( &cacheOccupancyContentionObserver, usage::getEmaIpc, - conf.getSectionOrNew(SIGNAL_DROP_ANALYZER_NAME), + conf.getSectionOrNew(SignalDropAnalyzer::NAME), Tag(QOS_CONTROLLER, "IPC detectorFilter"), Contention_Type_IPC), ipcEMAFilter( &ipcDropDetector, usage::getIpc, usage::setEmaIpc, - conf.getItemOrDefault(ema::ALPHA_IPC, - ema::DEFAULT_ALPHA_IPC), + ConfigValidator(conf.getValue(ALPHA_IPC_KEY)) + .getOrElse(ALPHA_IPC_DEFAULT), Tag(QOS_CONTROLLER, "ipcEMAFilter")), tooLowUsageFilter( &ipcEMAFilter, @@ -146,7 +126,9 @@ class CpuQoSPipeline : public QoSControllerPipeline { new CpuContentionStrategy( conf.getSectionOrNew(CpuContentionStrategy::NAME), usage::getEmaCpuUsage), - strategy::DEFAULT_CONTENTION_COOLDOWN, + ConfigValidator(conf.getValue( + QoSCorrectionObserver::CONTENTION_COOLDOWN_KEY)) + .getOrElse(CONTENTION_COOLDOWN_DEFAULT), Tag(QOS_CONTROLLER, CpuContentionStrategy::NAME)), overloadDetector( &cpuContentionObserver, @@ -157,8 +139,8 @@ class CpuQoSPipeline : public QoSControllerPipeline { &overloadDetector, usage::getCpuUsage, usage::setEmaCpuUsage, - conf.getItemOrDefault(ema::ALPHA_CPU, - ema::DEFAULT_ALPHA_CPU), + ConfigValidator(conf.getValue(ALPHA_CPU_KEY)) + .getOrElse(ALPHA_CPU_DEFAULT), Tag(QOS_CONTROLLER, "cpuEMAFilter")), cumulativeFilter( &tooLowUsageFilter, @@ -166,7 +148,8 @@ class CpuQoSPipeline : public QoSControllerPipeline { // First item in pipeline. For now, close the pipeline for QoS. valveFilter( &cumulativeFilter, - conf.getItemOrDefault(VALVE_OPENED, DEFAULT_VALVE_OPENED), + ConfigValidator(conf.getValue(VALVE_OPENED_KEY)) + .getOrElse(VALVE_OPENED_DEFAULT), Tag(QOS_CONTROLLER, "valveFilter")) { this->ageFilter.addConsumer(&valveFilter); // Setup starting producer. @@ -182,8 +165,23 @@ class CpuQoSPipeline : public QoSControllerPipeline { cumulativeFilter.addConsumer(&cpuEMAFilter); } + /** + * Alpha controls how long is the moving average period. + * The smaller alpha becomes, the longer your moving average is. + * It becomes smoother, but less reactive to new samples. + */ + static const constexpr char* ALPHA_KEY = "ALPHA"; + static const constexpr char* ALPHA_CPU_KEY = "ALPHA_CPU"; + static const constexpr char* ALPHA_IPC_KEY = "ALPHA_IPC"; + static const constexpr char* VALVE_OPENED_KEY = "VALVE_OPENED"; + private: - SerenityConfig conf; + Config initConf(const Config& _conf) { + return _conf; + } + + Config conf; + // --- Shared resource contention QoS CorrectionMergerFilter correctionMerger; @@ -203,6 +201,12 @@ class CpuQoSPipeline : public QoSControllerPipeline { ExecutorAgeFilter ageFilter; ValveFilter valveFilter; + + // Cfg + static const constexpr double_t ALPHA_IPC_DEFAULT = 0.9; + static const constexpr double_t ALPHA_CPU_DEFAULT = 0.9; + static const constexpr bool VALVE_OPENED_DEFAULT = true; + static constexpr int64_t CONTENTION_COOLDOWN_DEFAULT = 10; }; } // namespace serenity diff --git a/src/serenity/config.hpp b/src/serenity/config.hpp index 22b603c..70090d2 100644 --- a/src/serenity/config.hpp +++ b/src/serenity/config.hpp @@ -8,7 +8,6 @@ #include "boost/variant.hpp" -#include "serenity/default_vars.hpp" #include "serenity/serenity.hpp" #include "stout/option.hpp" @@ -17,57 +16,40 @@ namespace mesos { namespace serenity { + +/** + * Class which implement validation of the given value. + * In case of + */ template -class SerenityItem { +class ConfigValidator { public: - SerenityItem(T _value, T _defaultValue, std::string _key) : - value(_value), defaultValue(_defaultValue), key(_key) {}; + explicit ConfigValidator( + Result _value, Option _key = None()) + : value(_value), key(_key) {} - T getValueOrDefault() { - if (validationFailed) { + T getOrElse(T defaultValue) { + if (!value.isSome() || validationFailed) { return defaultValue; } - return value; + return value.get(); } - //! Validates that value is below given threshold item. - SerenityItem& validateValueIsBelow(SerenityItem _thresholdItem) { - assertTypeMatchNumericVariant(); - - if (value > _thresholdItem.getValueOrDefault()) { - LOG(WARNING) << key << " option which is " << value - << "must be below the " << _thresholdItem.key - << " which is " << _thresholdItem.getValueOrDefault(); - - validationFailed = true; - } - - return *this; - } //! Validates that value is below given threshold. - SerenityItem& validateValueIsBelow(T thresholdValue) { - assertTypeMatchNumericVariant(); - - if (value > thresholdValue) { - LOG(WARNING) << key << " option which is " << value - << "must be below " << thresholdValue; + ConfigValidator& validateValueIsBelow( + T thresholdValue, std::string additionalMsg = "") { + assertTypeMatchNumericVariant(); - validationFailed = true; + if (!value.isSome()) { + return *this; } - return *this; - } - - //! Validates that value is above given threshold item. - SerenityItem& validateValueIsAbove(SerenityItem _thresholdItem) { - assertTypeMatchNumericVariant(); - - if (value < _thresholdItem.getValueOrDefault()) { - LOG(WARNING) << key << " option which is " << value - << "must be above the " << _thresholdItem.key - << " which is " << _thresholdItem.getValueOrDefault(); + if (value.get() > thresholdValue) { + LOG(WARNING) << key.getOrElse("") << " option which is " << value.get() + << "must be below " << thresholdValue << ". " + << additionalMsg; validationFailed = true; } @@ -76,12 +58,18 @@ class SerenityItem { } //! Validates that value is above given threshold. - SerenityItem& validateValueIsAbove(T thresholdValue) { - assertTypeMatchNumericVariant(); + ConfigValidator& validateValueIsAbove( + T thresholdValue, std::string additionalMsg = "") { + assertTypeMatchNumericVariant(); + + if (!value.isSome()) { + return *this; + } - if (value < thresholdValue) { - LOG(WARNING) << key << " option which is " << value - << "must be above " << thresholdValue; + if (value.get() < thresholdValue) { + LOG(WARNING) << key.getOrElse("") << " option which is " << value.get() + << "must be above " << thresholdValue << ". " + << additionalMsg; validationFailed = true; } @@ -90,11 +78,15 @@ class SerenityItem { } //! Validates that value is positive. - SerenityItem& validateValueIsPositive() { - assertTypeMatchNumericVariant(); + ConfigValidator& validateValueIsPositive() { + assertTypeMatchNumericVariant(); - if (value < 0) { - LOG(WARNING) << key << " option which is " << value + if (!value.isSome()) { + return *this; + } + + if (value.get() < 0) { + LOG(WARNING) << key.getOrElse("") << " option which is " << value.get() << "must be above 0"; validationFailed = true; @@ -104,14 +96,13 @@ class SerenityItem { } protected: - T value; - const T defaultValue; - const std::string key; + Result value; + const Option key; + bool validationFailed = false; private: - template - static void assertTypeMatchNumericVariant() const { + static void assertTypeMatchNumericVariant() { static_assert(std::is_same() || std::is_same(), "Function supports only following numeric types: int64_t, " @@ -121,46 +112,47 @@ class SerenityItem { /** - * Serenity Config class which implements basic mechanism + * Config class which implements basic mechanism * to support specifying config parameters via string key map & sections. * * Check config_test.cpp to see example usage. */ -class SerenityConfig { +class Config { public: - SerenityConfig() {} + Config() {} /** - * Safe getter for item in config. - * Return default value in case of error or none. + * Getter for value in config. */ template - const SerenityItem getItemAndSetDefault( - const std::string& key, T defaultValue) const { + const Result getValue(const std::string& key) const { assertTypeMatchVariant(); - T result = defaultValue; + Result result = None(); // Get item from items map. - Option value = getVariantValue(key); + Option value = getVariantValue(key); if (value.isSome()) { // When item is found, try to parse it to the specified T type. try { result = boost::get(value.get()); } catch (std::exception& e) { - LOG(ERROR) << "Failed to parse " << key << " to type " - << typeid(T).name() << ". Field: " << e.what(); + std::stringstream ss; + ss << "Failed to parse " << key << " to type " + << typeid(T).name() << ". Field: " << e.what(); + LOG(ERROR) << ss.str(); + result = Error(ss.str()); } } - return SerenityItem(result, defaultValue, key); + return result; } /** * Gets config section. */ - Option getSection(const std::string& sectionKey) { + Option getSection(const std::string& sectionKey) { auto mapItem = this->sections.find(sectionKey); if (mapItem != this->sections.end()) { return *(mapItem->second); @@ -174,15 +166,15 @@ class SerenityConfig { * Gets config section. * In case there is not one, create empty section. */ - SerenityConfig& getSectionOrNew(const std::string& sectionKey) { + const Config& getSectionOrNew(const std::string& sectionKey) { auto mapItem = this->sections.find(sectionKey); if (mapItem != this->sections.end()) { return *(mapItem->second); } // In case of no section under this key - create empty section. - std::shared_ptr newSection = - std::shared_ptr(new SerenityConfig()); + std::shared_ptr newSection = + std::shared_ptr(new Config()); sections[sectionKey] = newSection; return *newSection; @@ -195,7 +187,7 @@ class SerenityConfig { /** * Overlapping custom configuration options using recursive copy. */ - void applyConfig(const SerenityConfig& customCfg) { + void applyConfig(const Config& customCfg) { recursiveCfgCopy(this, customCfg); } @@ -203,17 +195,17 @@ class SerenityConfig { /** * Variant type for storing multiple types of data in configuration. */ - using Item = boost::variant; + using Value = boost::variant; /** * Item */ - std::unordered_map items; + std::unordered_map items; /** * Support for hierarchical configuration sections. */ - std::unordered_map> sections; + std::unordered_map> sections; /** * Put config value for Item types. @@ -233,29 +225,37 @@ class SerenityConfig { /** * Put Item config value. */ - void putVariant(const std::string& key, SerenityConfig::Item value) { + void putVariant(const std::string& key, Value value) { this->items[key] = value; } +// /** +// * Put config section. +// */ +// virtual void putSection(const std::string& sectionKey, Config config) { +// this->sections[sectionKey] = config; +// } + /** * Recursive copy. */ - void recursiveCfgCopy(SerenityConfig* base, - const SerenityConfig& customCfg) const { + void recursiveCfgCopy(Config* base, + const Config& customCfg) const { for (auto customItem : customCfg.items) { base->items[customItem.first] = customItem.second; } for (auto customSection : customCfg.sections) { this->recursiveCfgCopy( - &(base->getSectionOrNew(customSection.first)), *customSection.second); + &(base->getSectionRefOrNew(customSection.first)), + *customSection.second); } } /** - * Getter for Item. + * Getter for Variant Value. */ - Option getVariantValue( + Option getVariantValue( const std::string& itemKey) const { auto mapItem = this->items.find(itemKey); if (mapItem != this->items.end()) { @@ -266,15 +266,33 @@ class SerenityConfig { return None(); } + /** + * Gets config section. + * In case there is not one, create empty section. + */ + Config& getSectionRefOrNew(const std::string& sectionKey) { + auto mapItem = this->sections.find(sectionKey); + if (mapItem != this->sections.end()) { + return *(mapItem->second); + } + + // In case of no section under this key - create empty section. + std::shared_ptr newSection = + std::shared_ptr(new Config()); + sections[sectionKey] = newSection; + + return *newSection; + } + private: template - static void assertTypeMatchVariant() const { + static void assertTypeMatchVariant() { static_assert(std::is_same() || std::is_same() || std::is_same() || std::is_same(), "Config supports only following types: bool, int64_t, " - "double_t, string"); + "double_t, string"); } }; diff --git a/src/serenity/default_vars.hpp b/src/serenity/default_vars.hpp deleted file mode 100644 index 22f4185..0000000 --- a/src/serenity/default_vars.hpp +++ /dev/null @@ -1,79 +0,0 @@ -#ifndef SERENITY_DEFAULT_VARS_HPP -#define SERENITY_DEFAULT_VARS_HPP - -#include -#include - -namespace mesos { -namespace serenity { - -namespace qos_pipeline { -const constexpr char* VALVE_OPENED = "VALVE_OPENED"; -constexpr bool DEFAULT_VALVE_OPENED = true; -const constexpr char* ENABLED_VISUALISATION = "ENABLED_VISUALISATION"; -constexpr bool DEFAULT_ENABLED_VISUALISATION = false; - -const constexpr char* ON_EMPTY_CORRECTION_INTERVAL = - "ON_EMPTY_CORRECTION_INTERVAL"; -constexpr double_t DEFAULT_ON_EMPTY_CORRECTION_INTERVAL = 2; - - -} // namespace qos_pipeline - - -namespace ema { -/** - * Alpha controls how long is the moving average period. - * The smaller alpha becomes, the longer your moving average is. - * It becomes smoother, but less reactive to new samples. - */ -const constexpr char* ALPHA = "ALPHA"; -constexpr double_t DEFAULT_ALPHA = 0.2; - -const constexpr char* ALPHA_CPU = "ALPHA_CPU"; -constexpr double_t DEFAULT_ALPHA_CPU = 0.9; -const constexpr char* ALPHA_IPC = "ALPHA_IPC"; -constexpr double_t DEFAULT_ALPHA_IPC = 0.9; - -} // namespace ema - -namespace detector { - -//const constexpr char* ANALYZER_TYPE = "ANALYZER_TYPE"; -constexpr double_t DEFAULT_START_VALUE = 0.00001; - -const constexpr char* THRESHOLD = "THRESHOLD"; -constexpr double_t DEFAULT_UTILIZATION_THRESHOLD = 0.72; -} // namespace detector - -namespace slack_observer { -constexpr double_t DEFAULT_MAX_OVERSUBSCRIPTION_FRACTION = 0.8; -} // namespace slack_observer - -namespace utilization { -constexpr double_t DEFAULT_THRESHOLD = 0.95; -} // namespace utilization - - -namespace new_executor { -constexpr uint32_t DEFAULT_THRESHOLD_SEC = 5 * 60; // !< Five minutes. -} // namespace new_executor - -namespace too_low_usage { -const constexpr char* MINIMAL_CPU_USAGE = "MINIMAL_CPU_USAGE"; -constexpr double_t DEFAULT_MINIMAL_CPU_USAGE = 0.25; // !< per sec. -} // namespace too_low_usage - -namespace strategy { -const constexpr char* CONTENTION_COOLDOWN = "CONTENTION_COOLDOWN"; -constexpr int64_t DEFAULT_CONTENTION_COOLDOWN = 10; -const constexpr char* DEFAULT_CPU_SEVERITY = "DEFAULT_CPU_SEVERITY"; -constexpr double_t DEFAULT_DEFAULT_CPU_SEVERITY = 1.0; -static const constexpr char* STARTING_SEVERITY = "STARTING_SEVERITY"; -constexpr double_t DEFAULT_STARTING_SEVERITY = 0.1; -} // namespace strategy - -} // namespace serenity -} // namespace mesos - -#endif // SERENITY_DEFAULT_VARS_HPP diff --git a/src/serenity/resource_helper.cpp b/src/serenity/resource_helper.cpp index 5d727b4..ce4f39a 100644 --- a/src/serenity/resource_helper.cpp +++ b/src/serenity/resource_helper.cpp @@ -50,7 +50,7 @@ const ResourceUsage_Executor& executor) { bool ResourceUsageHelper::isExecutorHasStatistics( const ResourceUsage_Executor& executor) { - return executor.has_executor_info() && executor.has_statistics(); + return executor.has_statistics(); } bool ResourceUsageHelper::isRevocableExecutor( diff --git a/src/tests/common/config_helper.hpp b/src/tests/common/config_helper.hpp deleted file mode 100644 index 443eec4..0000000 --- a/src/tests/common/config_helper.hpp +++ /dev/null @@ -1,42 +0,0 @@ -#ifndef SERENITY_TESTS_CONFIG_HELPER_HPP -#define SERENITY_TESTS_CONFIG_HELPER_HPP - -#include - -#include "serenity/config.hpp" - -namespace mesos { -namespace serenity { -namespace tests { - -inline SerenityConfig createAssuranceAnalyzerCfg( - const int64_t windowSize, - const int64_t maxCheckpoints, - const double_t fractionalThreshold, - const double_t severityLvl = detector::DEFAULT_SEVERITY_FRACTION, - const double_t nearLvl = detector::DEFAULT_NEAR_FRACTION, - const double_t quorum = detector::DEFAULT_QUORUM) { - SerenityConfig cfg; - cfg.set(detector::WINDOW_SIZE, windowSize); - cfg.set(detector::MAX_CHECKPOINTS, maxCheckpoints); - cfg.set(detector::FRACTIONAL_THRESHOLD, fractionalThreshold); - cfg.set(detector::SEVERITY_FRACTION, severityLvl); - cfg.set(detector::NEAR_FRACTION, nearLvl); - cfg.set(detector::QUORUM, quorum); - - return cfg; -} - -inline SerenityConfig createThresholdDetectorCfg( - const double_t utilization = detector::DEFAULT_UTILIZATION_THRESHOLD) { - SerenityConfig cfg; - cfg.set(detector::THRESHOLD, utilization); - - return cfg; -} - -} // namespace tests -} // namespace serenity -} // namespace mesos - -#endif // SERENITY_TESTS_CONFIG_HELPER_HPP diff --git a/src/tests/contention_detectors/overload_test.cpp b/src/tests/contention_detectors/overload_test.cpp index d60b8fe..edd7dfe 100644 --- a/src/tests/contention_detectors/overload_test.cpp +++ b/src/tests/contention_detectors/overload_test.cpp @@ -20,7 +20,6 @@ #include "stout/gtest.hpp" -#include "tests/common/config_helper.hpp" #include "tests/common/signal_helper.hpp" #include "tests/common/usage_helper.hpp" #include "tests/common/mocks/mock_sink.hpp" @@ -50,10 +49,9 @@ TEST(OverloadDetectorTest, LowUtilization) { OverloadDetector overloadDetector( &mockSink, usage::getCpuUsage, - SerenityConfig()); - - overloadDetector.setCfgUtilizationThreshold() + Config()); + overloadDetector.setUtilizationThreshold(UTIL_THRESHOLD); // Fake slave ResourceUsage source. MockSource usageSource(&overloadDetector); @@ -103,8 +101,9 @@ TEST(OverloadDetectorTest, HighUtilization) { OverloadDetector overloadDetector( &mockSink, usage::getCpuUsage, - createThresholdDetectorCfg( - UTIL_THRESHOLD)); + Config()); + + overloadDetector.setUtilizationThreshold(UTIL_THRESHOLD); // Fake slave ResourceUsage source. MockSource usageSource(&overloadDetector); @@ -159,9 +158,9 @@ TEST(OverloadDetectorTest, IntegrationTest) { OverloadDetector overloadDetector( &mockSink, usage::getEmaCpuUsage, - createThresholdDetectorCfg( - UTIL_THRESHOLD)); + Config()); + overloadDetector.setUtilizationThreshold(UTIL_THRESHOLD); EMAFilter cpuEMAFilter(&overloadDetector, usage::getCpuUsage, diff --git a/src/tests/contention_detectors/signal_analyzers/drop_test.cpp b/src/tests/contention_detectors/signal_analyzers/drop_test.cpp index 1251fd6..8d5d493 100644 --- a/src/tests/contention_detectors/signal_analyzers/drop_test.cpp +++ b/src/tests/contention_detectors/signal_analyzers/drop_test.cpp @@ -11,8 +11,6 @@ #include "serenity/data_utils.hpp" -#include "tests/common/config_helper.hpp" - namespace mesos { namespace serenity { namespace tests { @@ -35,12 +33,14 @@ TEST(SignalDropAnalyzerTest, StableSignal) { SignalDropAnalyzer signalDropAnalyzer( Tag(QOS_CONTROLLER, "SignalDropAnalyzer"), - createAssuranceAnalyzerCfg( - WINDOWS_SIZE, - MAX_CHECKPOINTS, - FRACTION_THRESHOLD, - SEVERITY_FRACTION, - NEAR_FRACTION)); + Config()); + + signalDropAnalyzer.setWindowsSizeAndMaxCheckpoints( + WINDOWS_SIZE, + MAX_CHECKPOINTS); + signalDropAnalyzer.setFractionalThreshold(FRACTION_THRESHOLD); + signalDropAnalyzer.setSeverityFraction(SEVERITY_FRACTION); + signalDropAnalyzer.setNearFraction(NEAR_FRACTION); SignalScenario signalGen = SignalScenario(ITERATIONS) @@ -67,13 +67,15 @@ TEST(SignalDropAnalyzerTest, StableLoadOneBigDrop) { SignalDropAnalyzer signalDropAnalyzer( Tag(QOS_CONTROLLER, "SignalDropAnalyzer"), - createAssuranceAnalyzerCfg( - WINDOWS_SIZE, - MAX_CHECKPOINTS, - FRACTION_THRESHOLD, - SEVERITY_FRACTION, - NEAR_FRACTION, - QUORUM)); + Config()); + + signalDropAnalyzer.setWindowsSizeAndMaxCheckpoints( + WINDOWS_SIZE, + MAX_CHECKPOINTS); + signalDropAnalyzer.setFractionalThreshold(FRACTION_THRESHOLD); + signalDropAnalyzer.setSeverityFraction(SEVERITY_FRACTION); + signalDropAnalyzer.setNearFraction(NEAR_FRACTION); + signalDropAnalyzer.setQuroumFraction(QUORUM); SignalScenario signalGen = SignalScenario(ITERATIONS) @@ -95,23 +97,25 @@ TEST(SignalDropAnalyzerTest, StableLoadOneBigDrop) { TEST(SignalDropAnalyzerTest, StableLoadOneBigDropWithReset) { - const uint64_t WINDOWS_SIZE = 8; - const uint64_t MAX_CHECKPOINTS = 4; + const int64_t WINDOWS_SIZE = 8; + const int64_t MAX_CHECKPOINTS = 4; const double_t FRACTION_THRESHOLD = 0.5; const double_t SEVERITY_FRACTION = 1; const double_t NEAR_FRACTION = 0.1; const double_t QUORUM = 0.50; - const uint64_t ITERATIONS = 30; + const int64_t ITERATIONS = 30; SignalDropAnalyzer signalDropAnalyzer( Tag(QOS_CONTROLLER, "SignalDropAnalyzer"), - createAssuranceAnalyzerCfg( - WINDOWS_SIZE, - MAX_CHECKPOINTS, - FRACTION_THRESHOLD, - SEVERITY_FRACTION, - NEAR_FRACTION, - QUORUM)); + Config()); + + signalDropAnalyzer.setWindowsSizeAndMaxCheckpoints( + WINDOWS_SIZE, + MAX_CHECKPOINTS); + signalDropAnalyzer.setFractionalThreshold(FRACTION_THRESHOLD); + signalDropAnalyzer.setSeverityFraction(SEVERITY_FRACTION); + signalDropAnalyzer.setNearFraction(NEAR_FRACTION); + signalDropAnalyzer.setQuroumFraction(QUORUM); SignalScenario signalGen = SignalScenario(ITERATIONS) @@ -146,13 +150,15 @@ TEST(SignalDropAnalyzerTest, StableLoadOneProgressiveDrop) { SignalDropAnalyzer signalDropAnalyzer( Tag(QOS_CONTROLLER, "SignalDropAnalyzer"), - createAssuranceAnalyzerCfg( - WINDOWS_SIZE, - MAX_CHECKPOINTS, - FRACTION_THRESHOLD, - SEVERITY_FRACTION, - NEAR_FRACTION, - QUORUM)); + Config()); + + signalDropAnalyzer.setWindowsSizeAndMaxCheckpoints( + WINDOWS_SIZE, + MAX_CHECKPOINTS); + signalDropAnalyzer.setFractionalThreshold(FRACTION_THRESHOLD); + signalDropAnalyzer.setSeverityFraction(SEVERITY_FRACTION); + signalDropAnalyzer.setNearFraction(NEAR_FRACTION); + signalDropAnalyzer.setQuroumFraction(QUORUM); SignalScenario signalGen = SignalScenario(ITERATIONS) @@ -184,13 +190,15 @@ TEST(SignalDropAnalyzerTest, StableLoadOneBigDropAndRecovery) { SignalDropAnalyzer signalDropAnalyzer( Tag(QOS_CONTROLLER, "SignalDropAnalyzer"), - createAssuranceAnalyzerCfg( - WINDOWS_SIZE, - MAX_CHECKPOINTS, - FRACTION_THRESHOLD, - SEVERITY_FRACTION, - NEAR_FRACTION, - QUORUM)); + Config()); + + signalDropAnalyzer.setWindowsSizeAndMaxCheckpoints( + WINDOWS_SIZE, + MAX_CHECKPOINTS); + signalDropAnalyzer.setFractionalThreshold(FRACTION_THRESHOLD); + signalDropAnalyzer.setSeverityFraction(SEVERITY_FRACTION); + signalDropAnalyzer.setNearFraction(NEAR_FRACTION); + signalDropAnalyzer.setQuroumFraction(QUORUM); SignalScenario signalGen = SignalScenario(ITERATIONS) @@ -225,13 +233,15 @@ TEST(SignalDropAnalyzerTest, NoisyLoadOneBigDropLessCheckpoints) { SignalDropAnalyzer signalDropAnalyzer( Tag(QOS_CONTROLLER, "SignalDropAnalyzer"), - createAssuranceAnalyzerCfg( - WINDOWS_SIZE, - MAX_CHECKPOINTS, - FRACTION_THRESHOLD, - SEVERITY_FRACTION, - NEAR_FRACTION, - QUORUM)); + Config()); + + signalDropAnalyzer.setWindowsSizeAndMaxCheckpoints( + WINDOWS_SIZE, + MAX_CHECKPOINTS); + signalDropAnalyzer.setFractionalThreshold(FRACTION_THRESHOLD); + signalDropAnalyzer.setSeverityFraction(SEVERITY_FRACTION); + signalDropAnalyzer.setNearFraction(NEAR_FRACTION); + signalDropAnalyzer.setQuroumFraction(QUORUM); SignalScenario signalGen = SignalScenario(ITERATIONS) @@ -265,13 +275,15 @@ TEST(SignalDropAnalyzerTest, NoisyLoadOneBigDropMoreCheckpoints) { SignalDropAnalyzer signalDropAnalyzer( Tag(QOS_CONTROLLER, "SignalDropAnalyzer"), - createAssuranceAnalyzerCfg( - WINDOWS_SIZE, - MAX_CHECKPOINTS, - FRACTION_THRESHOLD, - SEVERITY_FRACTION, - NEAR_FRACTION, - QUORUM)); + Config()); + + signalDropAnalyzer.setWindowsSizeAndMaxCheckpoints( + WINDOWS_SIZE, + MAX_CHECKPOINTS); + signalDropAnalyzer.setFractionalThreshold(FRACTION_THRESHOLD); + signalDropAnalyzer.setSeverityFraction(SEVERITY_FRACTION); + signalDropAnalyzer.setNearFraction(NEAR_FRACTION); + signalDropAnalyzer.setQuroumFraction(QUORUM); SignalScenario signalGen = SignalScenario(ITERATIONS) diff --git a/src/tests/mesos_modules/qos_controller/qos_controller_test.cpp b/src/tests/mesos_modules/qos_controller/qos_controller_test.cpp index 1109df7..6b737cc 100644 --- a/src/tests/mesos_modules/qos_controller/qos_controller_test.cpp +++ b/src/tests/mesos_modules/qos_controller/qos_controller_test.cpp @@ -51,7 +51,7 @@ class TestCorrectionPipeline : public QoSControllerPipeline { TEST(SerenityControllerTest, PipelineIntegration) { Try qoSController = serenity::SerenityController::create( - std::shared_ptr( + std::unique_ptr( new TestCorrectionPipeline())); ASSERT_SOME(qoSController); diff --git a/src/tests/observers/strategies/seniority_strategy_test.cpp b/src/tests/observers/strategies/seniority_strategy_test.cpp index bca7ab5..0820397 100644 --- a/src/tests/observers/strategies/seniority_strategy_test.cpp +++ b/src/tests/observers/strategies/seniority_strategy_test.cpp @@ -50,7 +50,7 @@ TEST(QoSCorrectionObserverSeniorityDeciderTest, EmptyContentions) { ExecutorAgeFilter age; QoSCorrectionObserver observer( - &mockSink, &age, new SeniorityStrategy(SerenityConfig())); + &mockSink, &age, new SeniorityStrategy(Config())); age.addConsumer(&observer); @@ -104,7 +104,7 @@ TEST(QoSCorrectionObserverSeniorityDeciderTest, OneContentionSmallSeverity) { ExecutorAgeFilter age; QoSCorrectionObserver observer( - &mockSink, &age, new SeniorityStrategy(SerenityConfig())); + &mockSink, &age, new SeniorityStrategy(Config())); age.addConsumer(&observer); diff --git a/src/tests/serenity/config_test.cpp b/src/tests/serenity/config_test.cpp index 82f9377..cddc424 100644 --- a/src/tests/serenity/config_test.cpp +++ b/src/tests/serenity/config_test.cpp @@ -5,8 +5,6 @@ #include "stout/gtest.hpp" -#include "tests/common/config_helper.hpp" - namespace mesos { namespace serenity { namespace tests { @@ -28,9 +26,7 @@ const constexpr char* FIELD_DOUBLE = "FIELD_DOUBLE"; const constexpr double_t DEFAULT_FIELD_DOUBLE = 0.345345; const constexpr double_t MODIFIED_FIELD_DOUBLE = 3.432; -const constexpr char* FIELD_SECTION = "SECTION1"; - -class TestConfig : SerenityConfig { +class TestConfig : Config { public: void loadSampleConfig() { put(FIELD_STR, (std::string) MODIFIED_FIELD_STR); @@ -39,90 +35,138 @@ class TestConfig : SerenityConfig { put(FIELD_DOUBLE, MODIFIED_FIELD_DOUBLE); } - void loadSampleConfigWithSections() { - SerenityConfig config = getSectionOrNew(FIELD_SECTION); + void loadSampleConfigWithSections(std::string sectionName) { + TestConfig config; config.put(FIELD_STR, (std::string) MODIFIED_FIELD_STR); + config.put(FIELD_BOOL, MODIFIED_FIELD_BOOL); + config.put(FIELD_INT, MODIFIED_FIELD_INT); + config.put(FIELD_DOUBLE, MODIFIED_FIELD_DOUBLE); + + Config& config2 = getSectionRefOrNew(sectionName); + config2.applyConfig(config); + } + + /** + * Put config value for Item types. + */ + template + void put(const std::string& key, T value) { + Config::putVariant(key, value); + } + + /** + * Getter for value in config. + */ + template + const Result getValue(const std::string& key) const { + return Config::getValue(key); + } + + const Config& getSectionOrNew(const std::string& sectionKey) { + return Config::getSectionOrNew(sectionKey); } }; -TEST(SerenityConfigTest, EmptyItemsTest) { - SerenityConfig config; - EXPECT_NONE(config.getItem(FIELD_STR)); - EXPECT_NONE(config.getItem(FIELD_BOOL)); - EXPECT_NONE(config.getItem(FIELD_INT)); - EXPECT_NONE(config.getItem(FIELD_DOUBLE)); +TEST(ConfigTest, EmptyItemsTest) { + Config config; + EXPECT_NONE(config.getValue(FIELD_STR)); + EXPECT_NONE(config.getValue(FIELD_BOOL)); + EXPECT_NONE(config.getValue(FIELD_INT)); + EXPECT_NONE(config.getValue(FIELD_DOUBLE)); } -TEST(SerenityConfigTest, DefaultItemsTest) { - SerenityConfig config; - EXPECT_EQ(config.getItemOrDefault(FIELD_STR, DEFAULT_FIELD_STR), - DEFAULT_FIELD_STR); - EXPECT_EQ(config.getItemOrDefault(FIELD_BOOL, DEFAULT_FIELD_BOOL), +TEST(ConfigTest, DefaultItemsTest) { + Config config; + EXPECT_EQ(ConfigValidator( + config.getValue(FIELD_STR)).getOrElse(DEFAULT_FIELD_STR), + DEFAULT_FIELD_STR); + EXPECT_EQ(ConfigValidator( + config.getValue(FIELD_BOOL)).getOrElse(DEFAULT_FIELD_BOOL), DEFAULT_FIELD_BOOL); - EXPECT_EQ(config.getItemOrDefault(FIELD_INT, DEFAULT_FIELD_INT), + EXPECT_EQ(ConfigValidator( + config.getValue(FIELD_INT)).getOrElse(DEFAULT_FIELD_INT), DEFAULT_FIELD_INT); - EXPECT_EQ(config.getItemOrDefault(FIELD_DOUBLE, DEFAULT_FIELD_DOUBLE), + EXPECT_EQ(ConfigValidator( + config.getValue(FIELD_DOUBLE)).getOrElse(DEFAULT_FIELD_DOUBLE), DEFAULT_FIELD_DOUBLE); } -TEST(SerenityConfigTest, ModifiedItemsTest) { +TEST(ConfigTest, ModifiedItemsTest) { TestConfig config; - EXPECT_EQ(config.getItemOrDefault(FIELD_STR, DEFAULT_FIELD_STR), + EXPECT_EQ(ConfigValidator( + config.getValue(FIELD_STR)).getOrElse(DEFAULT_FIELD_STR), DEFAULT_FIELD_STR); - EXPECT_EQ(config.getItemOrDefault(FIELD_BOOL, DEFAULT_FIELD_BOOL), + EXPECT_EQ(ConfigValidator( + config.getValue(FIELD_BOOL)).getOrElse(DEFAULT_FIELD_BOOL), DEFAULT_FIELD_BOOL); - EXPECT_EQ(config.getItemOrDefault(FIELD_INT, DEFAULT_FIELD_INT), + EXPECT_EQ(ConfigValidator( + config.getValue(FIELD_INT)).getOrElse(DEFAULT_FIELD_INT), DEFAULT_FIELD_INT); - EXPECT_EQ(config.getItemOrDefault(FIELD_DOUBLE, DEFAULT_FIELD_DOUBLE), + EXPECT_EQ(ConfigValidator( + config.getValue(FIELD_DOUBLE)).getOrElse(DEFAULT_FIELD_DOUBLE), DEFAULT_FIELD_DOUBLE); config.loadSampleConfig(); - EXPECT_EQ(config.getItemOrDefault(FIELD_STR, DEFAULT_FIELD_STR), + EXPECT_EQ(ConfigValidator( + config.getValue(FIELD_STR)).getOrElse(DEFAULT_FIELD_STR), MODIFIED_FIELD_STR); - EXPECT_EQ(config.getItemOrDefault(FIELD_BOOL, DEFAULT_FIELD_BOOL), + EXPECT_EQ(ConfigValidator( + config.getValue(FIELD_BOOL)).getOrElse(DEFAULT_FIELD_BOOL), MODIFIED_FIELD_BOOL); - EXPECT_EQ(config.getItemOrDefault(FIELD_INT, DEFAULT_FIELD_INT), + EXPECT_EQ(ConfigValidator( + config.getValue(FIELD_INT)).getOrElse(DEFAULT_FIELD_INT), MODIFIED_FIELD_INT); - EXPECT_EQ(config.getItemOrDefault(FIELD_DOUBLE, DEFAULT_FIELD_DOUBLE), + EXPECT_EQ(ConfigValidator( + config.getValue(FIELD_DOUBLE)).getOrElse(DEFAULT_FIELD_DOUBLE), MODIFIED_FIELD_DOUBLE); } -TEST(SerenityConfigTest, ErrorItemsTest) { +TEST(ConfigTest, ErrorItemsTest) { TestConfig config; config.loadSampleConfig(); - EXPECT_ERROR(config.getItem(FIELD_STR)); - EXPECT_ERROR(config.getItem(FIELD_BOOL)); - EXPECT_ERROR(config.getItem(FIELD_INT)); - EXPECT_ERROR(config.getItem(FIELD_DOUBLE)); + EXPECT_ERROR(config.getValue(FIELD_STR)); + EXPECT_ERROR(config.getValue(FIELD_BOOL)); + EXPECT_ERROR(config.getValue(FIELD_INT)); + EXPECT_ERROR(config.getValue(FIELD_DOUBLE)); } -TEST(SerenityConfigTest, ModifiedSectionItemsTest) { +TEST(ConfigTest, ModifiedSectionItemsTest) { + const constexpr char* FIELD_SECTION = "SECTION1"; TestConfig config; - EXPECT_EQ(config.getItemOrDefault(FIELD_STR, DEFAULT_FIELD_STR), + const Config& section = config.getSectionOrNew(FIELD_SECTION); + + EXPECT_EQ(ConfigValidator( + section.getValue(FIELD_STR)).getOrElse(DEFAULT_FIELD_STR), DEFAULT_FIELD_STR); - EXPECT_EQ(config.getItemOrDefault(FIELD_BOOL, DEFAULT_FIELD_BOOL), + EXPECT_EQ(ConfigValidator( + section.getValue(FIELD_BOOL)).getOrElse(DEFAULT_FIELD_BOOL), DEFAULT_FIELD_BOOL); - EXPECT_EQ(config.getItemOrDefault(FIELD_INT, DEFAULT_FIELD_INT), + EXPECT_EQ(ConfigValidator( + section.getValue(FIELD_INT)).getOrElse(DEFAULT_FIELD_INT), DEFAULT_FIELD_INT); - EXPECT_EQ(config.getItemOrDefault(FIELD_DOUBLE, DEFAULT_FIELD_DOUBLE), + EXPECT_EQ(ConfigValidator( + section.getValue(FIELD_DOUBLE)).getOrElse(DEFAULT_FIELD_DOUBLE), DEFAULT_FIELD_DOUBLE); - config.loadSampleConfig(); + config.loadSampleConfigWithSections(FIELD_SECTION); - EXPECT_EQ(config.getItemOrDefault(FIELD_STR, DEFAULT_FIELD_STR), + EXPECT_EQ(ConfigValidator( + section.getValue(FIELD_STR)).getOrElse(DEFAULT_FIELD_STR), MODIFIED_FIELD_STR); - EXPECT_EQ(config.getItemOrDefault(FIELD_BOOL, DEFAULT_FIELD_BOOL), + EXPECT_EQ(ConfigValidator( + section.getValue(FIELD_BOOL)).getOrElse(DEFAULT_FIELD_BOOL), MODIFIED_FIELD_BOOL); - EXPECT_EQ(config.getItemOrDefault(FIELD_INT, DEFAULT_FIELD_INT), + EXPECT_EQ(ConfigValidator( + section.getValue(FIELD_INT)).getOrElse(DEFAULT_FIELD_INT), MODIFIED_FIELD_INT); - EXPECT_EQ(config.getItemOrDefault(FIELD_DOUBLE, DEFAULT_FIELD_DOUBLE), + EXPECT_EQ(ConfigValidator( + section.getValue(FIELD_DOUBLE)).getOrElse(DEFAULT_FIELD_DOUBLE), MODIFIED_FIELD_DOUBLE); } - } // namespace tests } // namespace serenity } // namespace mesos