diff --git a/src/contention_detectors/overload.cpp b/src/contention_detectors/overload.cpp index 71648a2..902f803 100644 --- a/src/contention_detectors/overload.cpp +++ b/src/contention_detectors/overload.cpp @@ -5,39 +5,38 @@ #include "mesos/resources.hpp" +#include "serenity/resource_helper.hpp" + namespace mesos { namespace serenity { -Try OverloadDetector::consume(const ResourceUsage& in) { +const constexpr char* OverloadDetector::UTILIZATION_THRESHOLD_KEY; + +void OverloadDetector::allProductsReady() { Contentions product; + ResourceUsage usage = getConsumable().get(); - if (in.total_size() == 0) { - return Error(std::string(NAME) + " No total in ResourceUsage"); + if (usage.total_size() == 0) { + SERENITY_LOG(ERROR) << std::string(NAME) << " No total in ResourceUsage"; + produce(Contentions()); } - Resources totalAgentResources(in.total()); + Resources totalAgentResources(usage.total()); Option totalAgentCpus = totalAgentResources.cpus(); if (totalAgentCpus.isNone()) { - return Error(std::string(NAME) + " No total cpus in ResourceUsage"); + SERENITY_LOG(ERROR) << std::string(NAME) + << " No total cpus in ResourceUsage"; + produce(product); } double_t thresholdCpus = this->cfgUtilizationThreshold * totalAgentCpus.get(); double_t agentSumCpus = 0; uint64_t beExecutors = 0; - for (const ResourceUsage_Executor& inExec : in.executors()) { - if (!inExec.has_executor_info()) { - SERENITY_LOG(ERROR) << "Executor " - << " does not include executor_info"; - // Filter out these executors. - continue; - } - if (!inExec.has_statistics()) { - SERENITY_LOG(ERROR) << "Executor " - << inExec.executor_info().executor_id().value() - << " does not include statistics."; - // Filter out these executors. + for (const ResourceUsage_Executor& inExec : usage.executors()) { + // Validate for statistics and executor info. + if (!hasRequiredFields(inExec)) { continue; } @@ -49,13 +48,14 @@ Try OverloadDetector::consume(const ResourceUsage& in) { agentSumCpus += value.get(); - if (!Resources(inExec.allocated()).revocable().empty()) { + if (ResourceUsageHelper::isRevocableExecutor(inExec)) { beExecutors++; } } SERENITY_LOG(INFO) << "Sum = " << agentSumCpus << " vs total = " - << totalAgentCpus.get() << " [threshold = " << thresholdCpus << "]"; + << totalAgentCpus.get() << " [threshold = " + << thresholdCpus << "]"; if (agentSumCpus > thresholdCpus) { if (beExecutors == 0) { @@ -70,12 +70,25 @@ Try OverloadDetector::consume(const ResourceUsage& in) { } } - // Continue pipeline. - this->produce(product); - - return Nothing(); + produce(product); } +bool OverloadDetector::hasRequiredFields(const ResourceUsage_Executor& inExec) { + if (!inExec.has_executor_info()) { + SERENITY_LOG(ERROR) << "Executor " + << " does not include executor_info"; + return false; + } + + if (!inExec.has_statistics()) { + SERENITY_LOG(ERROR) << "Executor " + << inExec.executor_info().executor_id().value() + << " does not include statistics."; + return false; + } + + return true; +} } // namespace serenity } // namespace mesos diff --git a/src/contention_detectors/overload.hpp b/src/contention_detectors/overload.hpp index 305a7d1..6c2d118 100644 --- a/src/contention_detectors/overload.hpp +++ b/src/contention_detectors/overload.hpp @@ -20,22 +20,6 @@ namespace mesos { namespace serenity { -class OverloadDetectorConfig : public SerenityConfig { - public: - OverloadDetectorConfig() { } - - explicit OverloadDetectorConfig(const SerenityConfig& customCfg) { - this->initDefaults(); - this->applyConfig(customCfg); - } - - void initDefaults() { - //! double_t - //! Detector threshold. - this->fields[detector::THRESHOLD] = - detector::DEFAULT_UTILIZATION_THRESHOLD; - } -}; /** * OverloadDetector is able to create contention if utilization is above @@ -48,28 +32,40 @@ class OverloadDetector : OverloadDetector( Consumer* _consumer, const lambda::function& _cpuUsageGetFunction, - SerenityConfig _conf, + const Config& _conf, const Tag& _tag = Tag(QOS_CONTROLLER, NAME)) : tag(_tag), cpuUsageGetFunction(_cpuUsageGetFunction), Producer(_consumer) { - SerenityConfig config = OverloadDetectorConfig(_conf); - this->cfgUtilizationThreshold = - config.getD(detector::THRESHOLD); + // Parse config values. + setUtilizationThreshold( + _conf.getValue(UTILIZATION_THRESHOLD_KEY)); } ~OverloadDetector() {} - Try consume(const ResourceUsage& in) override; - static const constexpr char* NAME = "OverloadDetector"; + void setUtilizationThreshold(const Result& value) { + cfgUtilizationThreshold = + ConfigValidator(value, UTILIZATION_THRESHOLD_KEY) + .validateValueIsPositive() + .getOrElse(DEFAULT_UTILIZATION_THRESHOLD); + } + + static const constexpr char* UTILIZATION_THRESHOLD_KEY = "THRESHOLD"; + protected: + void allProductsReady() override; + bool hasRequiredFields(const ResourceUsage_Executor& inExec); + const Tag tag; const lambda::function cpuUsageGetFunction; // cfg parameters. double_t cfgUtilizationThreshold; + + static const constexpr double_t DEFAULT_UTILIZATION_THRESHOLD = 0.72; }; } // namespace serenity diff --git a/src/contention_detectors/signal_analyzers/drop.cpp b/src/contention_detectors/signal_analyzers/drop.cpp index 0fac13a..bcc7047 100644 --- a/src/contention_detectors/signal_analyzers/drop.cpp +++ b/src/contention_detectors/signal_analyzers/drop.cpp @@ -1,3 +1,4 @@ +#include #include #include @@ -10,6 +11,15 @@ namespace mesos { namespace serenity { + +const constexpr char* SignalDropAnalyzer::WINDOW_SIZE_KEY; +const constexpr char* SignalDropAnalyzer::FRACTIONAL_THRESHOLD_KEY; +const constexpr char* SignalDropAnalyzer::SEVERITY_FRACTION_KEY; +const constexpr char* SignalDropAnalyzer::NEAR_FRACTION_KEY; +const constexpr char* SignalDropAnalyzer::MAX_CHECKPOINTS_KEY; +const constexpr char* SignalDropAnalyzer::QUORUM_FRACTION_KEY; +const constexpr double_t SignalDropAnalyzer::START_VALUE_DEFAULT; + void SignalDropAnalyzer::shiftBasePoints() { for (std::list::iterator& basePoint : this->basePoints) { basePoint++; @@ -22,8 +32,7 @@ void SignalDropAnalyzer::recalculateParams() { this->window.clear(); this->basePoints.clear(); - uint64_t windowSize = this->cfgWindowSize; - + uint64_t windowSize = cfgWindowSize; // Find the biggest n in the T-2^n which fits within window length. uint64_t checkpoints = 0; while (windowSize > 0) { @@ -31,34 +40,31 @@ void SignalDropAnalyzer::recalculateParams() { checkpoints++; } - // Make sure it does not exceed MAX_CHECKPOINSs option. - if (checkpoints > this->cfgMaxCheckpoints) { - checkpoints = this->cfgMaxCheckpoints; - } + // Make sure it does not exceed MAX_CHECKPOINTS option. + checkpoints = std::min((int64_t)checkpoints, cfgMaxCheckpoints); // Get the Quorum number from QUORUM fraction parameter. - this->quorumNum = this->cfgQuroum * checkpoints; - if (this->quorumNum == 0 || this->quorumNum > checkpoints) { + quorumNum = cfgQuroumFraction * checkpoints; + if (quorumNum == 0 || quorumNum > checkpoints) { SERENITY_LOG(WARNING) << "Bad value for Quorum parameter. Creating 100%" << " quorum."; - this->quorumNum = checkpoints; + quorumNum = checkpoints; } std::stringstream checkpointLog; - checkpointLog << "Assurance Parameters: Quorum = " - << this->quorumNum << "/" + checkpointLog << "Assurance Parameters: Quorum = " << quorumNum << "/" << checkpoints << " Checkpoints [ "; // Iterate over window and initialize it. Choose proper base points starting // from the end of window. uint64_t choosenNum = pow(2, (--checkpoints)); - for (uint64_t i = this->cfgWindowSize; i > 0 ; i--) { - this->window.push_back(detector::DEFAULT_START_VALUE); + for (uint64_t i = cfgWindowSize; i > 0 ; i--) { + window.push_back(START_VALUE_DEFAULT); if (choosenNum == i) { checkpointLog << "T-" << choosenNum << " "; choosenNum /= 2; - basePoints.push_back(--this->window.end()); + basePoints.push_back(--window.end()); } } checkpointLog << "]"; @@ -68,6 +74,12 @@ void SignalDropAnalyzer::recalculateParams() { Result SignalDropAnalyzer::processSample(double_t in) { + if (paramsChanged) { + recalculateParams(); + paramsChanged = false; + } + + // Fill window. if (in < 0.1) in = 0.1; diff --git a/src/contention_detectors/signal_analyzers/drop.hpp b/src/contention_detectors/signal_analyzers/drop.hpp index c423572..988f0cd 100644 --- a/src/contention_detectors/signal_analyzers/drop.hpp +++ b/src/contention_detectors/signal_analyzers/drop.hpp @@ -14,7 +14,6 @@ #include "messages/serenity.hpp" #include "serenity/config.hpp" -#include "serenity/default_vars.hpp" #include "serenity/data_utils.hpp" #include "serenity/executor_map.hpp" #include "serenity/executor_set.hpp" @@ -28,58 +27,6 @@ namespace mesos { namespace serenity { -#define SIGNAL_DROP_ANALYZER_NAME "AssuranceDropAnalyzer" - -class SignalDropAnalyzerConfig : public SerenityConfig { - public: - SignalDropAnalyzerConfig() {} - - explicit SignalDropAnalyzerConfig(const SerenityConfig& customCfg) { - this->initDefaults(); - this->applyConfig(customCfg); - } - - void initDefaults() { - this->fields[detector::ANALYZER_TYPE] = SIGNAL_DROP_ANALYZER_NAME; - //! uint64_t - //! How far in the past we look. - this->fields[detector::WINDOW_SIZE] = - detector::DEFAULT_WINDOW_SIZE; - - //! double_t - //! Defines how much (relatively to base point) value must drop to trigger - //! contention. - //! Most signal_analyzer will use that. - this->fields[detector::FRACTIONAL_THRESHOLD] = - detector::DEFAULT_FRACTIONAL_THRESHOLD; - - //! double_t - //! You can adjust how big severity is created for a defined drop. - //! if -1 then unknown severity will be reported. - this->fields[detector::SEVERITY_FRACTION] = (double_t) -1; - - //! double_t - //! Tolerance fraction of threshold if signal is accepted as returned to - //! previous state after drop. - this->fields[detector::NEAR_FRACTION] = - detector::DEFAULT_NEAR_FRACTION; - - //! uint64_t - //! Maximum number of checkpoints we will have in our assurance detector. - //! Checkpoints are the reference assurance_test(base) points which we refer - //! to in the past when detecting drop or not. - //! It needs to be 0 < < WINDOW_SIZE - this->fields[detector::MAX_CHECKPOINTS] = - detector::DEFAULT_MAX_CHECKPOINTS; - - //! double_t - //! Fraction of checkpoints' votes that important decision needs to obtain. - this->fields[detector::QUORUM] = - detector::DEFAULT_QUORUM; - } -}; - - /** * Dynamic implementation of sequential change point detection. * @@ -105,19 +52,18 @@ class SignalDropAnalyzer : public SignalAnalyzer { public: explicit SignalDropAnalyzer( const Tag& _tag, - const SerenityConfig& _config) + const Config& _config) : SignalAnalyzer(_tag), valueBeforeDrop(None()), quorumNum(0) { - SerenityConfig config = SignalDropAnalyzerConfig(_config); - this->cfgWindowSize = config.getU64(detector::WINDOW_SIZE); - this->cfgMaxCheckpoints = config.getU64(detector::MAX_CHECKPOINTS); - this->cfgQuroum = config.getD(detector::QUORUM); - this->cfgFractionalThreshold = config.getD(detector::FRACTIONAL_THRESHOLD); - this->cfgNearFraction = config.getD(detector::NEAR_FRACTION); - this->cfgSeverityFraction = config.getD(detector::SEVERITY_FRACTION); - - this->recalculateParams(); + setWindowsSizeAndMaxCheckpoints( + _config.getValue(WINDOW_SIZE_KEY), + _config.getValue(MAX_CHECKPOINTS_KEY)); + setQuroumFraction(_config.getValue(QUORUM_FRACTION_KEY)); + setFractionalThreshold( + _config.getValue(FRACTIONAL_THRESHOLD_KEY)); + setNearFraction(_config.getValue(NEAR_FRACTION_KEY)); + setSeverityFraction(_config.getValue(SEVERITY_FRACTION_KEY)); } Result _processSample(double_t in); @@ -131,28 +77,102 @@ class SignalDropAnalyzer : public SignalAnalyzer { */ void shiftBasePoints(); + //! WindowSize: How far in the past we look. + //! MaxCheckpoints: Maximum number of checkpoints we will have in our + //! assurance detector. + //! Checkpoints are the reference assurance_test(base) points which we refer + //! to in the past when detecting drop or not. + //! It needs to be 0 < and < WINDOW_SIZE. + void setWindowsSizeAndMaxCheckpoints( + const Result& _cfgWindowSize, + const Result& _cfgMaxCheckpoints) { + cfgWindowSize = ConfigValidator(_cfgWindowSize, WINDOW_SIZE_KEY) + .validateValueIsPositive() + .getOrElse(WINDOW_SIZE_DEFAULT); + + cfgMaxCheckpoints = + ConfigValidator(_cfgMaxCheckpoints, MAX_CHECKPOINTS_KEY) + .validateValueIsPositive() + .validateValueIsBelow(cfgWindowSize, WINDOW_SIZE_KEY) + .getOrElse(MAX_CHECKPOINTS_DEFAULT); + paramsChanged = true; + } + + //! Fraction of checkpoints' votes needed to make a Drop contention. + void setQuroumFraction(const Result& value) { + cfgQuroumFraction = ConfigValidator(value, QUORUM_FRACTION_KEY) + .validateValueIsPositive() + .getOrElse(QUORUM_FRACTION_DEFAULT); + paramsChanged = true; + } + + //! Defines how much (relatively to base point) value must drop to trigger + //! contention. + void setFractionalThreshold(const Result& value) { + cfgFractionalThreshold = + ConfigValidator(value, FRACTIONAL_THRESHOLD_KEY) + .validateValueIsPositive() + .getOrElse(FRACTIONAL_THRESHOLD_DEFAULT); + paramsChanged = true; + } + + //! You can adjust how big severity is created for a defined drop. + //! if -1 then unknown severity will be reported. + void setSeverityFraction(const Result& value) { + cfgSeverityFraction = + ConfigValidator(value, SEVERITY_FRACTION_KEY) + .getOrElse(SEVERITY_FRACTION_DEFAULT); + paramsChanged = true; + } + + //! Tolerance fraction of threshold if signal is accepted as returned to + //! previous state after drop. + void setNearFraction(const Result& value) { + cfgNearFraction = + ConfigValidator(value, NEAR_FRACTION_KEY) + .validateValueIsPositive() + .getOrElse(NEAR_FRACTION_DEFAULT); + paramsChanged = true; + } + + static const constexpr char* NAME = "SignalDropAnalyzer"; + static const constexpr char* WINDOW_SIZE_KEY = "WINDOW_SIZE"; + static const constexpr char* FRACTIONAL_THRESHOLD_KEY = + "FRACTIONAL_THRESHOLD"; + static const constexpr char* SEVERITY_FRACTION_KEY = "SEVERITY_FRACTION"; + static const constexpr char* NEAR_FRACTION_KEY = "NEAR_FRACTION"; + static const constexpr char* MAX_CHECKPOINTS_KEY = "MAX_CHECKPOINTS"; + static const constexpr char* QUORUM_FRACTION_KEY = "QUORUM_FRACTION"; + protected: + void recalculateParams(); + std::list window; std::list::iterator> basePoints; // If none then there was no drop. Option valueBeforeDrop; - uint32_t dropVotes; - uint32_t quorumNum; + uint64_t dropVotes; + uint64_t quorumNum; + bool paramsChanged = true; - // cfg parameters. - uint64_t cfgWindowSize; - uint64_t cfgMaxCheckpoints; - double_t cfgQuroum; + // Cfg parameters. + int64_t cfgWindowSize; + int64_t cfgMaxCheckpoints; + double_t cfgQuroumFraction; double_t cfgFractionalThreshold; double_t cfgSeverityFraction; double_t cfgNearFraction; - /** - * It is possible to dynamically change analyzer configuration. - */ - void recalculateParams(); + // Cfg default values. + static const constexpr int64_t WINDOW_SIZE_DEFAULT = 10; + static constexpr double_t FRACTIONAL_THRESHOLD_DEFAULT = 0.3; + static constexpr double_t SEVERITY_FRACTION_DEFAULT = 2.1; + static constexpr double_t NEAR_FRACTION_DEFAULT = 0.1; + static constexpr int64_t MAX_CHECKPOINTS_DEFAULT = 3; + static constexpr double_t QUORUM_FRACTION_DEFAULT = 0.7; + static constexpr double_t START_VALUE_DEFAULT = 0.00001; }; diff --git a/src/contention_detectors/signal_based.hpp b/src/contention_detectors/signal_based.hpp index 412330c..cc5aa4a 100644 --- a/src/contention_detectors/signal_based.hpp +++ b/src/contention_detectors/signal_based.hpp @@ -42,7 +42,7 @@ class SignalBasedDetector : SignalBasedDetector( Consumer* _consumer, const lambda::function& _getValue, - SerenityConfig _detectorConf, + const Config& _detectorConf, const Tag& _tag = Tag(QOS_CONTROLLER, "SignalBasedDetector"), const Contention_Type _contentionType = Contention_Type_IPC) : tag(_tag), @@ -65,7 +65,7 @@ class SignalBasedDetector : // Detections. ExecutorMap> detectors; - SerenityConfig detectorConf; + const Config detectorConf; }; } // namespace serenity diff --git a/src/filters/ema.hpp b/src/filters/ema.hpp index d3e294c..d406aae 100644 --- a/src/filters/ema.hpp +++ b/src/filters/ema.hpp @@ -10,7 +10,6 @@ #include "messages/serenity.hpp" #include "serenity/data_utils.hpp" -#include "serenity/default_vars.hpp" #include "serenity/executor_map.hpp" #include "serenity/executor_set.hpp" #include "serenity/serenity.hpp" @@ -36,7 +35,7 @@ class ExponentialMovingAverage { public: ExponentialMovingAverage( EMASeriesType _seriesType = EMA_REGULAR_SERIES, - double_t _alpha = ema::DEFAULT_ALPHA) + double_t _alpha = ALPHA_DEFAULT) : alpha(_alpha), seriesType(_seriesType), uninitialized(true) {} @@ -55,6 +54,7 @@ class ExponentialMovingAverage { double_t calculateEMA(double_t sample, double_t sampleTimestamp); private: + static const constexpr double_t ALPHA_DEFAULT = 0.2; //! Constant describing how the window weights decrease over time. //! It controls how long the moving average period is. //! The smaller alpha becomes, the longer your moving average is. @@ -103,7 +103,7 @@ class EMAFilter : Consumer* _consumer, const lambda::function& _valueGetFunction, const lambda::function& _valueSetFunction, - double_t _alpha = ema::DEFAULT_ALPHA, + double_t _alpha = ALPHA_DEFAULT, const Tag& _tag = Tag(UNDEFINED, "emaFilter")) : tag(_tag), Producer(_consumer), emaSamples(new ExecutorMap()), @@ -121,6 +121,8 @@ class EMAFilter : const lambda::function valueGetFunction; const lambda::function valueSetFunction; std::unique_ptr> emaSamples; + + static const constexpr double_t ALPHA_DEFAULT = 0.2; }; } // namespace serenity diff --git a/src/filters/ignore_new_executors.hpp b/src/filters/ignore_new_executors.hpp index eb0d150..34285ed 100644 --- a/src/filters/ignore_new_executors.hpp +++ b/src/filters/ignore_new_executors.hpp @@ -9,7 +9,6 @@ #include "messages/serenity.hpp" -#include "serenity/default_vars.hpp" #include "serenity/executor_map.hpp" #include "serenity/serenity.hpp" @@ -32,7 +31,7 @@ class IgnoreNewExecutorsFilter : public Consumer, public: explicit IgnoreNewExecutorsFilter( Consumer* _consumer = nullptr, - uint32_t _thresholdSeconds = new_executor::DEFAULT_THRESHOLD_SEC) : + uint32_t _thresholdSeconds = THRESHOLD_SEC_DEFAULT) : Producer(_consumer), threshold(_thresholdSeconds), executorTimestamps(new ExecutorMap) {} @@ -61,6 +60,8 @@ class IgnoreNewExecutorsFilter : public Consumer, static constexpr const char* name = "[SerenityEstimator] IgnoreNewExecutorsFilter: "; + + static constexpr uint32_t THRESHOLD_SEC_DEFAULT = 5 * 60; // !< Five minutes. }; } // namespace serenity diff --git a/src/filters/too_low_usage.cpp b/src/filters/too_low_usage.cpp index 66154c3..48cae01 100644 --- a/src/filters/too_low_usage.cpp +++ b/src/filters/too_low_usage.cpp @@ -18,6 +18,9 @@ using std::map; using std::pair; using std::string; + +const constexpr char* TooLowUsageFilter::MINIMAL_CPU_USAGE_KEY; + TooLowUsageFilter::~TooLowUsageFilter() {} @@ -58,11 +61,11 @@ Try TooLowUsageFilter::consume(const ResourceUsage& in) { continue; } - if (cpuUsage.get() <= this->cfgMinimalCpuUsage) { + if (cpuUsage.get() <= cfgMinimalCpuUsage) { // Exclude executor. SERENITY_LOG(INFO) << "Filtering out PR exec: " << executor_id << " because of its CPU Usage: " << cpuUsage.get() << " [Min: " - << this->cfgMinimalCpuUsage << "]"; + << cfgMinimalCpuUsage << "]"; continue; } } diff --git a/src/filters/too_low_usage.hpp b/src/filters/too_low_usage.hpp index 63a5fdb..824e178 100644 --- a/src/filters/too_low_usage.hpp +++ b/src/filters/too_low_usage.hpp @@ -14,24 +14,6 @@ namespace mesos { namespace serenity { -class TooLowUsageFilterConfig : public SerenityConfig { - public: - TooLowUsageFilterConfig() { } - - explicit TooLowUsageFilterConfig(const SerenityConfig& customCfg) { - this->initDefaults(); - this->applyConfig(customCfg); - } - - void initDefaults() { - //! double_t - //! Minimal cpu usage - this->fields[too_low_usage::MINIMAL_CPU_USAGE] = - too_low_usage::DEFAULT_MINIMAL_CPU_USAGE; - } -}; - - /** * Filter out PR executors with too low metrics. * Currently we filter out when CPU Usage is below specified threshold. @@ -44,23 +26,35 @@ class TooLowUsageFilter : explicit TooLowUsageFilter( Consumer* _consumer, - SerenityConfig _conf, + const Config& _conf, const Tag& _tag = Tag(QOS_CONTROLLER, NAME)) : Producer(_consumer), tag(_tag) { - SerenityConfig config = TooLowUsageFilterConfig(_conf); - this->cfgMinimalCpuUsage = config.getD(too_low_usage::MINIMAL_CPU_USAGE); + setMinimalCpuUsage(_conf.getValue(MINIMAL_CPU_USAGE_KEY)); } ~TooLowUsageFilter(); - static const constexpr char* NAME = "TooLowUsageFilter"; + Try consume(const ResourceUsage& in); - public: + void setMinimalCpuUsage(Result cfgMinimalCpuUsage) { + cfgMinimalCpuUsage = + ConfigValidator(cfgMinimalCpuUsage, MINIMAL_CPU_USAGE_KEY) + .validateValueIsPositive() + .getOrElse(MINIMAL_CPU_USAGE_DEFAULT); + } + + static const constexpr char* NAME = "TooLowUsageFilter"; + + static const constexpr char* MINIMAL_CPU_USAGE_KEY = "MINIMAL_CPU_USAGE"; + + protected: const Tag tag; double_t cfgMinimalCpuUsage; + + static constexpr double_t MINIMAL_CPU_USAGE_DEFAULT = 0.25; // !< per sec. }; } // namespace serenity diff --git a/src/filters/utilization_threshold.hpp b/src/filters/utilization_threshold.hpp index df0ae5e..b7097a9 100644 --- a/src/filters/utilization_threshold.hpp +++ b/src/filters/utilization_threshold.hpp @@ -4,7 +4,6 @@ #include #include -#include "serenity/default_vars.hpp" #include "serenity/executor_set.hpp" #include "serenity/serenity.hpp" @@ -27,7 +26,7 @@ class UtilizationThresholdFilter : public Consumer, public Producer { public: UtilizationThresholdFilter( - double_t _utilizationThreshold = utilization::DEFAULT_THRESHOLD, + double_t _utilizationThreshold = THRESHOLD_DEFAULT, const Tag& _tag = Tag(UNDEFINED, "utilizationFilter")) : tag(_tag), utilizationThreshold(_utilizationThreshold), @@ -35,7 +34,7 @@ class UtilizationThresholdFilter : UtilizationThresholdFilter( Consumer* _consumer, - double_t _utilizationThreshold = utilization::DEFAULT_THRESHOLD, + double_t _utilizationThreshold = THRESHOLD_DEFAULT, const Tag& _tag = Tag(UNDEFINED, "utilizationFilter")) : tag(_tag), Producer(_consumer), utilizationThreshold(_utilizationThreshold), @@ -56,6 +55,8 @@ class UtilizationThresholdFilter : const std::string UTILIZATION_THRESHOLD_FILTER_WARNING = "Filter is not" \ "able to calculate total cpu usage and will base on allocated " \ " resources to cut off oversubscription if needed."; + + static constexpr double_t THRESHOLD_DEFAULT = 0.95; }; } // namespace serenity diff --git a/src/mesos_modules/qos_controller/serenity_controller.cpp b/src/mesos_modules/qos_controller/serenity_controller.cpp index 1678130..26e554e 100644 --- a/src/mesos_modules/qos_controller/serenity_controller.cpp +++ b/src/mesos_modules/qos_controller/serenity_controller.cpp @@ -32,10 +32,10 @@ class SerenityControllerProcess : public: SerenityControllerProcess( const lambda::function()>& _usage, - std::shared_ptr _pipeline, + std::unique_ptr _pipeline, double _onEmptyCorrectionInterval) : usage(_usage), - pipeline(_pipeline), + pipeline(std::move(_pipeline)), onEmptyCorrectionInterval(_onEmptyCorrectionInterval) {} Future corrections() { @@ -84,7 +84,7 @@ class SerenityControllerProcess : private: const lambda::function()> usage; - std::shared_ptr pipeline; + std::unique_ptr pipeline; //! How much time we wait in case of empty correction. //! This value should be near the perf interval since it is useless //! to rerun QoS pipeline on the same perf's counter collection. @@ -109,7 +109,7 @@ Try SerenityController::initialize( } process.reset(new SerenityControllerProcess( - usage, this->pipeline, this->onEmptyCorrectionInterval)); + usage, std::move(this->pipeline), this->onEmptyCorrectionInterval)); spawn(process.get()); return Nothing(); diff --git a/src/mesos_modules/qos_controller/serenity_controller.hpp b/src/mesos_modules/qos_controller/serenity_controller.hpp index acf6cea..c31ff6b 100644 --- a/src/mesos_modules/qos_controller/serenity_controller.hpp +++ b/src/mesos_modules/qos_controller/serenity_controller.hpp @@ -26,15 +26,16 @@ class SerenityControllerProcess; class SerenityController: public slave::QoSController { public: explicit SerenityController( - std::shared_ptr _pipeline, + std::unique_ptr _pipeline, double _onEmptyCorrectionInterval) - : pipeline(_pipeline), + : pipeline(std::move(_pipeline)), onEmptyCorrectionInterval(_onEmptyCorrectionInterval) {} static Try create( - std::shared_ptr _pipeline, + std::unique_ptr _pipeline, double _onEmptyCorrectionInterval = 5) { - return new SerenityController(_pipeline, _onEmptyCorrectionInterval); + return new SerenityController(std::move(_pipeline), + _onEmptyCorrectionInterval); } virtual ~SerenityController(); @@ -46,7 +47,7 @@ class SerenityController: public slave::QoSController { protected: process::Owned process; - std::shared_ptr pipeline; + std::unique_ptr pipeline; double onEmptyCorrectionInterval; }; diff --git a/src/mesos_modules/qos_controller/serenity_controller_module.cpp b/src/mesos_modules/qos_controller/serenity_controller_module.cpp index 82cc197..7bc05ea 100644 --- a/src/mesos_modules/qos_controller/serenity_controller_module.cpp +++ b/src/mesos_modules/qos_controller/serenity_controller_module.cpp @@ -17,15 +17,11 @@ // TODO(nnielsen): Should be explicit using-directives. using namespace mesos; // NOLINT(build/namespaces) -using namespace mesos::serenity::ema; // NOLINT(build/namespaces) -using namespace mesos::serenity::strategy; // NOLINT(build/namespaces) -using namespace mesos::serenity::detector; // NOLINT(build/namespaces) -using namespace mesos::serenity::too_low_usage; // NOLINT(build/namespaces) -using namespace mesos::serenity::qos_pipeline; // NOLINT(build/namespaces) using mesos::serenity::CpuContentionStrategy; using mesos::serenity::CpuQoSPipeline; -using mesos::serenity::SerenityConfig; +using mesos::serenity::Config; +using mesos::serenity::ConfigValidator; using mesos::serenity::SerenityController; using mesos::serenity::SeniorityStrategy; using mesos::serenity::SignalBasedDetector; @@ -34,53 +30,28 @@ using mesos::serenity::QoSControllerPipeline; using mesos::slave::QoSController; +const constexpr char* ON_EMPTY_CORRECTION_INTERVAL_KEY = + "ON_EMPTY_CORRECTION_INTERVAL"; + +const constexpr double_t ON_EMPTY_CORRECTION_INTERVAL_DEFAULT = 2; // IPC QoS pipeline. static QoSController* createSerenityController( const Parameters& parameters) { LOG(INFO) << "Loading Serenity QoS Controller module"; // TODO(bplotka): Fetch configuration from parameters or conf file. - // - // --Hardcoded configuration for Serenity QoS Controller--- - - SerenityConfig conf; - // AssuranceDropAnalyzer configuration: - // How far we look back in samples. - conf[SIGNAL_DROP_ANALYZER_NAME].set(WINDOW_SIZE, (uint64_t) 10); - // Defines how much (relatively to base point) value must drop to trigger - // contention. - // Most signal_analyzer will use that. - conf[SIGNAL_DROP_ANALYZER_NAME].set(FRACTIONAL_THRESHOLD, (double_t) 0.3); - conf[SIGNAL_DROP_ANALYZER_NAME].set(SEVERITY_FRACTION, (double_t) 2.1); - - // How many iterations observers will wait with creating another - // correction. - conf[CpuContentionStrategy::NAME].set(CONTENTION_COOLDOWN, (uint64_t) 10); - conf[SeniorityStrategy::NAME].set(CONTENTION_COOLDOWN, (uint64_t) 10); - - // UtilizationDetector configuration: - // CPU utilization threshold. - conf[THRESHOLD].set(THRESHOLD, (double_t) 0.72); - - conf[TooLowUsageFilter::NAME].set(MINIMAL_CPU_USAGE, (double_t) 0.25); - - conf.set(ALPHA_CPU, (double_t) 0.9); - conf.set(ALPHA_IPC, (double_t) 0.9); - conf.set(ENABLED_VISUALISATION, false); - conf.set(VALVE_OPENED, true); - - // Since slave is configured for 5 second perf interval, it is useless to - // check correction more often then 5 sec. - double onEmptyCorrectionInterval = 2; + Config conf; - // --End of hardcoded configuration for Serenity QoS Controller--- + double_t onEmptyCorrectionInterval = + ConfigValidator( + conf.getValue(ON_EMPTY_CORRECTION_INTERVAL_KEY)) + .getOrElse(ON_EMPTY_CORRECTION_INTERVAL_DEFAULT); // Use static constructor of QoSController. Try result = - SerenityController::create( - std::shared_ptr( - new CpuQoSPipeline(conf)), - onEmptyCorrectionInterval); + SerenityController::create(std::unique_ptr( + new CpuQoSPipeline(conf)), + onEmptyCorrectionInterval); if (result.isError()) { return NULL; diff --git a/src/observers/qos_correction.cpp b/src/observers/qos_correction.cpp index 8bc1d98..c946e5e 100644 --- a/src/observers/qos_correction.cpp +++ b/src/observers/qos_correction.cpp @@ -57,7 +57,7 @@ void QoSCorrectionObserver::allProductsReady() { // Strategy has pointed aggressors, so don't pass // current contentions to next QoS Controller. - iterationCooldownCounter = this->cooldownIterations; + iterationCooldownCounter = cfgCooldownIterations; produceResults(corrections.get(), Contentions()); } diff --git a/src/observers/qos_correction.hpp b/src/observers/qos_correction.hpp index 7b43510..4ddb4f3 100644 --- a/src/observers/qos_correction.hpp +++ b/src/observers/qos_correction.hpp @@ -51,13 +51,13 @@ class QoSCorrectionObserver : public Consumer, Consumer* _consumer, ExecutorAgeFilter* _ageFilter = new ExecutorAgeFilter(), RevocationStrategy* _revStrategy = - new SeniorityStrategy(SerenityConfig()), - uint32_t _cooldownIterations = strategy::DEFAULT_CONTENTION_COOLDOWN, + new SeniorityStrategy(Config()), + int64_t _cooldownIterations = CONTENTION_COOLDOWN_DEFAULT, const Tag& _tag = Tag(QOS_CONTROLLER, NAME)) : Producer(_consumer), revocationStrategy(_revStrategy), executorAgeFilter(_ageFilter), - cooldownIterations(_cooldownIterations), + cfgCooldownIterations(_cooldownIterations), tag(_tag) {} ~QoSCorrectionObserver(); @@ -69,6 +69,8 @@ class QoSCorrectionObserver : public Consumer, static constexpr const char* NAME = "QoSCorrectionObserver"; + static const constexpr char* CONTENTION_COOLDOWN_KEY = "CONTENTION_COOLDOWN"; + protected: void allProductsReady() override; @@ -111,7 +113,9 @@ class QoSCorrectionObserver : public Consumer, /** * Default iterationCooldownCounter start value. */ - uint64_t cooldownIterations; + int64_t cfgCooldownIterations; + + static constexpr int64_t CONTENTION_COOLDOWN_DEFAULT = 10; }; } // namespace serenity diff --git a/src/observers/slack_resource.hpp b/src/observers/slack_resource.hpp index 3883971..c9abad2 100644 --- a/src/observers/slack_resource.hpp +++ b/src/observers/slack_resource.hpp @@ -11,7 +11,6 @@ #include "stout/result.hpp" -#include "serenity/default_vars.hpp" #include "serenity/executor_set.hpp" #include "serenity/serenity.hpp" @@ -39,16 +38,16 @@ class SlackResourceObserver : public Consumer, public: explicit SlackResourceObserver( double_t _maxOversubscriptionFraction = - slack_observer::DEFAULT_MAX_OVERSUBSCRIPTION_FRACTION) - : previousSamples(new ExecutorSet()), - maxOversubscriptionFraction(_maxOversubscriptionFraction), - default_role(getDefaultRole()) {} + MAX_OVERSUBSCRIPTION_FRACTION_DEFAULT) + : previousSamples(new ExecutorSet()), + maxOversubscriptionFraction(_maxOversubscriptionFraction), + default_role(getDefaultRole()) {} SlackResourceObserver( Consumer* _consumer, double_t _maxOversubscriptionFraction = - slack_observer::DEFAULT_MAX_OVERSUBSCRIPTION_FRACTION) : - Producer(_consumer), + MAX_OVERSUBSCRIPTION_FRACTION_DEFAULT) + : Producer(_consumer), maxOversubscriptionFraction(_maxOversubscriptionFraction), previousSamples(new ExecutorSet()), default_role(getDefaultRole()) {} @@ -76,6 +75,8 @@ class SlackResourceObserver : public Consumer, SlackResourceObserver(const SlackResourceObserver& other) : default_role(getDefaultRole()) {} std::string default_role; + + static const constexpr double_t MAX_OVERSUBSCRIPTION_FRACTION_DEFAULT = 0.8; }; } // namespace serenity diff --git a/src/observers/strategies/cache_occupancy.hpp b/src/observers/strategies/cache_occupancy.hpp index 1fe7e58..777338b 100644 --- a/src/observers/strategies/cache_occupancy.hpp +++ b/src/observers/strategies/cache_occupancy.hpp @@ -28,7 +28,7 @@ class CacheOccupancyStrategy : public RevocationStrategy { init(); } - explicit CacheOccupancyStrategy(const SerenityConfig& _config) + explicit CacheOccupancyStrategy(const Config& _config) : RevocationStrategy(Tag(QOS_CONTROLLER, NAME)) { init(); } @@ -42,7 +42,7 @@ class CacheOccupancyStrategy : public RevocationStrategy { protected: void init() { - minimalCacheOccupancy = DEFAULT_MINIMAL_CACHE_OCCUPANCY; + minimalCacheOccupancy = MINIMAL_CACHE_OCCUPANCY_DEFAULT; } std::vector getCmtEnabledExecutors( @@ -56,7 +56,7 @@ class CacheOccupancyStrategy : public RevocationStrategy { const double_t meanCacheOccupancy) const; //!< Minimal cache occupancy for executor to be revoked. - static constexpr uint64_t DEFAULT_MINIMAL_CACHE_OCCUPANCY = 1000000; // 1M + static constexpr uint64_t MINIMAL_CACHE_OCCUPANCY_DEFAULT = 1000000; // 1M uint64_t minimalCacheOccupancy; }; diff --git a/src/observers/strategies/cpu_contention.cpp b/src/observers/strategies/cpu_contention.cpp index caa6674..3d607e4 100644 --- a/src/observers/strategies/cpu_contention.cpp +++ b/src/observers/strategies/cpu_contention.cpp @@ -16,6 +16,8 @@ namespace serenity { using std::list; using std::pair; +const constexpr char* CpuContentionStrategy::DEFAULT_CPU_SEVERITY_KEY; + Try CpuContentionStrategy::decide( ExecutorAgeFilter* ageFilter, const Contentions& currentContentions, @@ -43,7 +45,7 @@ Try CpuContentionStrategy::decide( if (contention.has_severity()) { cpuToRecover = std::max(contention.severity(), cpuToRecover); } else { - cpuToRecover = std::max(this->defaultSeverity, cpuToRecover); + cpuToRecover = std::max(cfgDefaultSeverity, cpuToRecover); } } diff --git a/src/observers/strategies/cpu_contention.hpp b/src/observers/strategies/cpu_contention.hpp index de3c840..789e823 100644 --- a/src/observers/strategies/cpu_contention.hpp +++ b/src/observers/strategies/cpu_contention.hpp @@ -12,30 +12,6 @@ namespace mesos { namespace serenity { -class CpuContentionStrategyConfig : public SerenityConfig { - public: - CpuContentionStrategyConfig() { - this->initDefaults(); - } - - explicit CpuContentionStrategyConfig(const SerenityConfig& customCfg) { - this->initDefaults(); - this->applyConfig(customCfg); - } - - void initDefaults() { - // uint64_t - // Specify the initial value of iterations we should wait until - // we create new correction. - this->fields[strategy::CONTENTION_COOLDOWN] = - strategy::DEFAULT_CONTENTION_COOLDOWN; - // double_t - this->fields[strategy::DEFAULT_CPU_SEVERITY] = - strategy::DEFAULT_DEFAULT_CPU_SEVERITY; - } -}; - - /** * Checks contentions and choose executors to kill. * It accepts only Contention_Type_CPU. @@ -47,25 +23,35 @@ class CpuContentionStrategyConfig : public SerenityConfig { class CpuContentionStrategy : public RevocationStrategy { public: explicit CpuContentionStrategy( - const SerenityConfig& _config, + const Config& _config, const lambda::function& _cpuUsageGetFunction) : RevocationStrategy(Tag(QOS_CONTROLLER, "CpuContentionStrategy")), getCpuUsage(_cpuUsageGetFunction) { - SerenityConfig config = CpuContentionStrategyConfig(_config); + setDefaultSeverity(_config.getValue(DEFAULT_CPU_SEVERITY_KEY)); } Try decide(ExecutorAgeFilter* ageFilter, const Contentions& currentContentions, const ResourceUsage& currentUsage); + void setDefaultSeverity(const Result value) { + cfgDefaultSeverity = + ConfigValidator(value, DEFAULT_CPU_SEVERITY_KEY) + .getOrElse(CPU_SEVERITY_DEFAULT); + } + static const constexpr char* NAME = "CpuContentionStrategy"; + static const constexpr char* DEFAULT_CPU_SEVERITY_KEY = + "DEFAULT_CPU_SEVERITY"; + private: const lambda::function getCpuUsage; // cfg parameters. - uint64_t cooldownTime; - double_t defaultSeverity; + double_t cfgDefaultSeverity; + + static constexpr double_t CPU_SEVERITY_DEFAULT = 1.0; }; } // namespace serenity diff --git a/src/observers/strategies/seniority.cpp b/src/observers/strategies/seniority.cpp index 06c21a7..cc8e3a8 100644 --- a/src/observers/strategies/seniority.cpp +++ b/src/observers/strategies/seniority.cpp @@ -15,6 +15,7 @@ using std::list; using std::pair; +const constexpr char* SeniorityStrategy::STARTING_SEVERITY_KEY; Try SeniorityStrategy::decide( ExecutorAgeFilter* ageFilter, diff --git a/src/observers/strategies/seniority.hpp b/src/observers/strategies/seniority.hpp index 4e72b70..0abd563 100644 --- a/src/observers/strategies/seniority.hpp +++ b/src/observers/strategies/seniority.hpp @@ -20,37 +20,31 @@ namespace serenity { */ class SeniorityStrategy : public RevocationStrategy { public: - SeniorityStrategy() : RevocationStrategy(Tag(QOS_CONTROLLER, NAME)) { - initialize(); - } - /** * TODO(skonefal): SerenityConfig should have const methods inside. * Currently, it cannot be passed as const. */ - explicit SeniorityStrategy(SerenityConfig _config) + explicit SeniorityStrategy(const Config& _config) : RevocationStrategy(Tag(QOS_CONTROLLER, NAME)) { - initialize(); - if (_config.hasKey(STARTING_SEVERITY_KEY)) { - severity = _config.getD(STARTING_SEVERITY_KEY); - } + setSeverity(_config.getValue(STARTING_SEVERITY_KEY)); } Try decide(ExecutorAgeFilter*, const Contentions&, const ResourceUsage&); + void setSeverity(Result value) { + severity = ConfigValidator(value, STARTING_SEVERITY_KEY) + .getOrElse(SEVERITY_DEFAULT); + } + static const constexpr char* STARTING_SEVERITY_KEY = "STARTING_SEVERITY"; static const constexpr char* NAME = "SeniorityStrategy"; private: - void initialize() { - severity = DEFAULT_SEVERITY; - } - - static const constexpr double_t DEFAULT_SEVERITY = 0.1; - double_t severity; + + static const constexpr double_t SEVERITY_DEFAULT = 0.1; }; } // namespace serenity diff --git a/src/pipeline/estimator_pipeline.hpp b/src/pipeline/estimator_pipeline.hpp index b11c994..b2c922c 100644 --- a/src/pipeline/estimator_pipeline.hpp +++ b/src/pipeline/estimator_pipeline.hpp @@ -13,7 +13,6 @@ #include "pipeline/pipeline.hpp" -#include "serenity/default_vars.hpp" #include "serenity/serenity.hpp" #include "time_series_export/slack_ts_export.hpp" @@ -56,8 +55,8 @@ using ResourceEstimatorPipeline = Pipeline; class CpuEstimatorPipeline : public ResourceEstimatorPipeline { public: explicit CpuEstimatorPipeline( - double_t _newExecutorsThreshold = new_executor::DEFAULT_THRESHOLD_SEC, - double_t _utilizationThreshold = utilization::DEFAULT_THRESHOLD, + double_t _newExecutorsThreshold = THRESHOLD_SEC_DEFAULT, + double_t _utilizationThreshold = THRESHOLD_DEFAULT, bool _visualisation = false, bool _valveOpened = true) : // Time series exporters. @@ -101,6 +100,9 @@ class CpuEstimatorPipeline : public ResourceEstimatorPipeline { PrExecutorPassFilter prExecutorPassFilter; UtilizationThresholdFilter utilizationFilter; ValveFilter valveFilter; + + static constexpr double_t THRESHOLD_DEFAULT = 0.95; + static constexpr uint32_t THRESHOLD_SEC_DEFAULT = 5 * 60; // !< Five minutes. }; } // namespace serenity diff --git a/src/pipeline/qos_pipeline.hpp b/src/pipeline/qos_pipeline.hpp index 9e16812..d3d62da 100644 --- a/src/pipeline/qos_pipeline.hpp +++ b/src/pipeline/qos_pipeline.hpp @@ -28,33 +28,9 @@ #include "serenity/data_utils.hpp" #include "serenity/serenity.hpp" -#include "time_series_export/resource_usage_ts_export.hpp" - namespace mesos { namespace serenity { -using namespace qos_pipeline; // NOLINT(build/namespaces) - -class QoSPipelineConfig : public SerenityConfig { - public: - QoSPipelineConfig() {} - - explicit QoSPipelineConfig(const SerenityConfig& customCfg) { - this->initDefaults(); - this->applyConfig(customCfg); - } - - void initDefaults() { - // Used sections: QoSCorrectionObserver, AssuranceDetector, - // UtilizationDetector - // TODO(bplotka): Move EMA conf to separate section. - this->fields[ema::ALPHA] = ema::DEFAULT_ALPHA; - this->fields[VALVE_OPENED] = DEFAULT_VALVE_OPENED; - this->fields[ENABLED_VISUALISATION] = DEFAULT_ENABLED_VISUALISATION; - } -}; - - using QoSControllerPipeline = Pipeline; @@ -104,11 +80,8 @@ using QoSControllerPipeline = Pipeline; */ class CpuQoSPipeline : public QoSControllerPipeline { public: - explicit CpuQoSPipeline(const SerenityConfig& _conf) - : conf(QoSPipelineConfig(_conf)), - // Time series exporters. - rawResourcesExporter("raw"), - emaFilteredResourcesExporter("ema"), + explicit CpuQoSPipeline(const Config& _conf) + : conf(_conf), // NOTE(bplotka): age Filter should initialized first before passing // to the qosCorrectionObserver. ageFilter(), @@ -116,52 +89,58 @@ class CpuQoSPipeline : public QoSControllerPipeline { correctionMerger( this, Tag(QOS_CONTROLLER, "CorrectionMerger")), -// ipcContentionObserver( -// &correctionMerger, -// &ageFilter, -// new SeniorityStrategy(conf[SeniorityStrategy::NAME]), -// strategy::DEFAULT_CONTENTION_COOLDOWN, -// Tag(QOS_CONTROLLER, SeniorityStrategy::NAME)), + // ipcContentionObserver( + // &correctionMerger, + // &ageFilter, + // new SeniorityStrategy(conf[SeniorityStrategy::NAME]), + // strategy::DEFAULT_CONTENTION_COOLDOWN, + // Tag(QOS_CONTROLLER, SeniorityStrategy::NAME)), cacheOccupancyContentionObserver( &correctionMerger, &ageFilter, new CacheOccupancyStrategy(), - strategy::DEFAULT_CONTENTION_COOLDOWN, + ConfigValidator(conf.getValue( + QoSCorrectionObserver::CONTENTION_COOLDOWN_KEY)) + .getOrElse(CONTENTION_COOLDOWN_DEFAULT), Tag(QOS_CONTROLLER, CacheOccupancyStrategy::NAME)), ipcDropDetector( &cacheOccupancyContentionObserver, usage::getEmaIpc, - conf[SIGNAL_DROP_ANALYZER_NAME], + conf.getSectionOrNew(SignalDropAnalyzer::NAME), Tag(QOS_CONTROLLER, "IPC detectorFilter"), Contention_Type_IPC), ipcEMAFilter( &ipcDropDetector, usage::getIpc, usage::setEmaIpc, - conf.getD(ema::ALPHA_IPC), + ConfigValidator(conf.getValue(ALPHA_IPC_KEY)) + .getOrElse(ALPHA_IPC_DEFAULT), Tag(QOS_CONTROLLER, "ipcEMAFilter")), tooLowUsageFilter( &ipcEMAFilter, - conf[TooLowUsageFilter::NAME], + conf.getSectionOrNew(TooLowUsageFilter::NAME), Tag(QOS_CONTROLLER, "tooLowCPUUsageFilter")), cpuContentionObserver( &correctionMerger, &ageFilter, new CpuContentionStrategy( - conf[CpuContentionStrategy::NAME], + conf.getSectionOrNew(CpuContentionStrategy::NAME), usage::getEmaCpuUsage), - strategy::DEFAULT_CONTENTION_COOLDOWN, + ConfigValidator(conf.getValue( + QoSCorrectionObserver::CONTENTION_COOLDOWN_KEY)) + .getOrElse(CONTENTION_COOLDOWN_DEFAULT), Tag(QOS_CONTROLLER, CpuContentionStrategy::NAME)), overloadDetector( &cpuContentionObserver, usage::getEmaCpuUsage, - conf[OverloadDetector::NAME], + conf.getSectionOrNew(OverloadDetector::NAME), Tag(QOS_CONTROLLER, "CPU High Usage utilization detector")), cpuEMAFilter( &overloadDetector, usage::getCpuUsage, usage::setEmaCpuUsage, - conf.getD(ema::ALPHA_CPU), + ConfigValidator(conf.getValue(ALPHA_CPU_KEY)) + .getOrElse(ALPHA_CPU_DEFAULT), Tag(QOS_CONTROLLER, "cpuEMAFilter")), cumulativeFilter( &tooLowUsageFilter, @@ -169,34 +148,40 @@ class CpuQoSPipeline : public QoSControllerPipeline { // First item in pipeline. For now, close the pipeline for QoS. valveFilter( &cumulativeFilter, - conf.getB(VALVE_OPENED), + ConfigValidator(conf.getValue(VALVE_OPENED_KEY)) + .getOrElse(VALVE_OPENED_DEFAULT), Tag(QOS_CONTROLLER, "valveFilter")) { this->ageFilter.addConsumer(&valveFilter); // Setup starting producer. this->addConsumer(&ageFilter); -// cacheOccupancyContentionObserver. -// Producer::addConsumer(&ipcContentionObserver); + // cacheOccupancyContentionObserver. + // Producer::addConsumer(&ipcContentionObserver); // QoSCorrection observers needs ResourceUsage as well. cpuEMAFilter.addConsumer(&cpuContentionObserver); -// cumulativeFilter.addConsumer(&ipcContentionObserver); + // cumulativeFilter.addConsumer(&ipcContentionObserver); cumulativeFilter.addConsumer(&cacheOccupancyContentionObserver); cumulativeFilter.addConsumer(&cpuEMAFilter); - - // Setup Time Series export - if (conf.getB(ENABLED_VISUALISATION)) { - this->addConsumer(&rawResourcesExporter); - ipcEMAFilter.addConsumer(&emaFilteredResourcesExporter); - } } + /** + * Alpha controls how long is the moving average period. + * The smaller alpha becomes, the longer your moving average is. + * It becomes smoother, but less reactive to new samples. + */ + static const constexpr char* ALPHA_KEY = "ALPHA"; + static const constexpr char* ALPHA_CPU_KEY = "ALPHA_CPU"; + static const constexpr char* ALPHA_IPC_KEY = "ALPHA_IPC"; + static const constexpr char* VALVE_OPENED_KEY = "VALVE_OPENED"; + private: - SerenityConfig conf; + Config initConf(const Config& _conf) { + return _conf; + } + + Config conf; - // --- Time Series Exporters --- - ResourceUsageTimeSeriesExporter rawResourcesExporter; - ResourceUsageTimeSeriesExporter emaFilteredResourcesExporter; // --- Shared resource contention QoS CorrectionMergerFilter correctionMerger; @@ -216,6 +201,12 @@ class CpuQoSPipeline : public QoSControllerPipeline { ExecutorAgeFilter ageFilter; ValveFilter valveFilter; + + // Cfg + static const constexpr double_t ALPHA_IPC_DEFAULT = 0.9; + static const constexpr double_t ALPHA_CPU_DEFAULT = 0.9; + static const constexpr bool VALVE_OPENED_DEFAULT = true; + static constexpr int64_t CONTENTION_COOLDOWN_DEFAULT = 10; }; } // namespace serenity diff --git a/src/serenity/config.hpp b/src/serenity/config.hpp index 3b40105..70090d2 100644 --- a/src/serenity/config.hpp +++ b/src/serenity/config.hpp @@ -4,191 +4,261 @@ #include #include #include +#include #include "boost/variant.hpp" -#include "serenity/default_vars.hpp" #include "serenity/serenity.hpp" #include "stout/option.hpp" +#include "stout/result.hpp" namespace mesos { namespace serenity { + /** - * Global Serenity Config class which implements basic mechanism - * to support specifying config parameters via string key map. - * - * Check config_test.cpp to see example usage. - * - * TODO(skonefal): every getter should pack result in Try. + * Class which implement validation of the given value. + * In case of */ -class SerenityConfig { +template +class ConfigValidator { public: - SerenityConfig() {} + explicit ConfigValidator( + Result _value, Option _key = None()) + : value(_value), key(_key) {} - /** - * Variant type for storing multiple types of data in configuration. - */ - using CfgVariant = boost::variant< - bool, int64_t, uint64_t, double_t, std::string>; + T getOrElse(T defaultValue) { + if (!value.isSome() || validationFailed) { + return defaultValue; + } - /** - * Overlapping custom configuration options using recursive copy. - */ - void applyConfig(const SerenityConfig& customCfg) { - this->recursiveCfgCopy(this, customCfg); + return value.get(); } - /** - * Gets variant config value. - */ - Option operator()(std::string key) const { - return getField(key); + + //! Validates that value is below given threshold. + ConfigValidator& validateValueIsBelow( + T thresholdValue, std::string additionalMsg = "") { + assertTypeMatchNumericVariant(); + + if (!value.isSome()) { + return *this; + } + + if (value.get() > thresholdValue) { + LOG(WARNING) << key.getOrElse("") << " option which is " << value.get() + << "must be below " << thresholdValue << ". " + << additionalMsg; + + validationFailed = true; + } + + return *this; } - /** - * Gets config section. - * In case there is not one, create empty section. - */ - SerenityConfig& operator[](std::string key) { - return *getSection(key); + //! Validates that value is above given threshold. + ConfigValidator& validateValueIsAbove( + T thresholdValue, std::string additionalMsg = "") { + assertTypeMatchNumericVariant(); + + if (!value.isSome()) { + return *this; + } + + if (value.get() < thresholdValue) { + LOG(WARNING) << key.getOrElse("") << " option which is " << value.get() + << "must be above " << thresholdValue << ". " + << additionalMsg; + + validationFailed = true; + } + + return *this; } - // -- unsafe getters -- + //! Validates that value is positive. + ConfigValidator& validateValueIsPositive() { + assertTypeMatchNumericVariant(); - /** - * Unsafe getter for string - */ - std::string getS(std::string key) { - return boost::get(this->fields[key]); + if (!value.isSome()) { + return *this; + } + + if (value.get() < 0) { + LOG(WARNING) << key.getOrElse("") << " option which is " << value.get() + << "must be above 0"; + + validationFailed = true; + } + + return *this; } - /** - * Unsafe getter for int64_t - */ - int64_t getI64(std::string key) { - return boost::get(this->fields[key]); + protected: + Result value; + const Option key; + + bool validationFailed = false; + + private: + static void assertTypeMatchNumericVariant() { + static_assert(std::is_same() + || std::is_same(), + "Function supports only following numeric types: int64_t, " + "double_t"); } +}; + + +/** + * Config class which implements basic mechanism + * to support specifying config parameters via string key map & sections. + * + * Check config_test.cpp to see example usage. + */ +class Config { + public: + Config() {} /** - * Unsafe getter for uint64_t + * Getter for value in config. */ - uint64_t getU64(std::string key) { - return boost::get(this->fields[key]); + template + const Result getValue(const std::string& key) const { + assertTypeMatchVariant(); + + Result result = None(); + + // Get item from items map. + Option value = getVariantValue(key); + + if (value.isSome()) { + // When item is found, try to parse it to the specified T type. + try { + result = boost::get(value.get()); + } catch (std::exception& e) { + std::stringstream ss; + ss << "Failed to parse " << key << " to type " + << typeid(T).name() << ". Field: " << e.what(); + LOG(ERROR) << ss.str(); + result = Error(ss.str()); + } + } + + return result; } /** - * Unsafe getter for double_t + * Gets config section. */ - double_t getD(std::string key) { - return boost::get(this->fields[key]); + Option getSection(const std::string& sectionKey) { + auto mapItem = this->sections.find(sectionKey); + if (mapItem != this->sections.end()) { + return *(mapItem->second); + } + + // Element not found. + return None(); } /** - * Unsafe getter for bool + * Gets config section. + * In case there is not one, create empty section. */ - bool getB(std::string key) { - return boost::get(this->fields[key]); - } + const Config& getSectionOrNew(const std::string& sectionKey) { + auto mapItem = this->sections.find(sectionKey); + if (mapItem != this->sections.end()) { + return *(mapItem->second); + } - // -- setters -- + // In case of no section under this key - create empty section. + std::shared_ptr newSection = + std::shared_ptr(new Config()); + sections[sectionKey] = newSection; - /** - * Sets char* config value. - */ - void set(std::string key, char* value) { - this->setVariant(key, (std::string)value); + return *newSection; } - /** - * Sets string config value. - */ - void set(std::string key, std::string value) { - this->setVariant(key, value); + bool hasKey(const std::string& key) const { + return items.find(key) != items.end(); } /** - * Sets bool config value. - */ - void set(std::string key, bool value) { - this->setVariant(key, value); + * Overlapping custom configuration options using recursive copy. + */ + void applyConfig(const Config& customCfg) { + recursiveCfgCopy(this, customCfg); } + protected: /** - * Sets uint64_t config value. + * Variant type for storing multiple types of data in configuration. */ - void set(std::string key, uint64_t value) { - this->setVariant(key, value); - } + using Value = boost::variant; /** - * Sets int64_t config value. + * Item */ - void set(std::string key, int64_t value) { - this->setVariant(key, value); - } + std::unordered_map items; /** - * Sets double_t config value. + * Support for hierarchical configuration sections. */ - void set(std::string key, double_t value) { - this->setVariant(key, value); - } + std::unordered_map> sections; /** - * Sets CfgVariant config value. + * Put config value for Item types. */ - void setVariant(std::string key, SerenityConfig::CfgVariant value) { - this->fields[key] = value; - } - - bool hasKey(std::string key) { - return fields.find(key) != fields.end(); + template + void put(const std::string& key, T value) { + this->putVariant(key, value); } /** - * TODO(skonefal): Add UT for usage of this enum. + * Put config value for char*. */ - enum ConfigurationType : int { - BOOL = 0, - INT64 = 1, - UINT64 = 2, - DOUBLE = 3, - STRING = 4 - }; - - protected: - std::unordered_map fields; + void put(const std::string& key, char* value) { + this->putVariant(key, (std::string) value); + } /** - * Support for hierarchical configuration sections. + * Put Item config value. */ - std::unordered_map> sections; + void putVariant(const std::string& key, Value value) { + this->items[key] = value; + } + +// /** +// * Put config section. +// */ +// virtual void putSection(const std::string& sectionKey, Config config) { +// this->sections[sectionKey] = config; +// } /** - * Getter for section. - * In case of no section - create such. + * Recursive copy. */ - std::shared_ptr getSection(std::string sectionKey) { - auto mapItem = this->sections.find(sectionKey); - if (mapItem != this->sections.end()) { - return mapItem->second; + void recursiveCfgCopy(Config* base, + const Config& customCfg) const { + for (auto customItem : customCfg.items) { + base->items[customItem.first] = customItem.second; } - // In case of no section under this key - create empty section. - auto newSection = std::make_shared(SerenityConfig()); - this->sections[sectionKey] = newSection; - - return newSection; + for (auto customSection : customCfg.sections) { + this->recursiveCfgCopy( + &(base->getSectionRefOrNew(customSection.first)), + *customSection.second); + } } /** - * Getter for field. + * Getter for Variant Value. */ - Option getField(std::string fieldKey) const { - auto mapItem = this->fields.find(fieldKey); - if (mapItem != this->fields.end()) { + Option getVariantValue( + const std::string& itemKey) const { + auto mapItem = this->items.find(itemKey); + if (mapItem != this->items.end()) { return mapItem->second; } @@ -197,18 +267,32 @@ class SerenityConfig { } /** - * Recursive copy. - */ - void recursiveCfgCopy(SerenityConfig* base, - const SerenityConfig& customCfg) const { - for (auto customItem : customCfg.fields) { - base->fields[customItem.first] = customItem.second; + * Gets config section. + * In case there is not one, create empty section. + */ + Config& getSectionRefOrNew(const std::string& sectionKey) { + auto mapItem = this->sections.find(sectionKey); + if (mapItem != this->sections.end()) { + return *(mapItem->second); } - for (auto customSection : customCfg.sections) { - this->recursiveCfgCopy( - &((*base)[customSection.first]), *customSection.second); - } + // In case of no section under this key - create empty section. + std::shared_ptr newSection = + std::shared_ptr(new Config()); + sections[sectionKey] = newSection; + + return *newSection; + } + + private: + template + static void assertTypeMatchVariant() { + static_assert(std::is_same() + || std::is_same() + || std::is_same() + || std::is_same(), + "Config supports only following types: bool, int64_t, " + "double_t, string"); } }; diff --git a/src/serenity/default_vars.hpp b/src/serenity/default_vars.hpp deleted file mode 100644 index bfd6a47..0000000 --- a/src/serenity/default_vars.hpp +++ /dev/null @@ -1,83 +0,0 @@ -#ifndef SERENITY_DEFAULT_VARS_HPP -#define SERENITY_DEFAULT_VARS_HPP - -#include -#include - -namespace mesos { -namespace serenity { - -namespace qos_pipeline { -const constexpr char* VALVE_OPENED = "VALVE_OPENED"; -constexpr bool DEFAULT_VALVE_OPENED = true; -const constexpr char* ENABLED_VISUALISATION = "ENABLED_VISUALISATION"; -constexpr bool DEFAULT_ENABLED_VISUALISATION = true; -} // namespace qos_pipeline - - -namespace ema { -/** - * Alpha controls how long is the moving average period. - * The smaller alpha becomes, the longer your moving average is. - * It becomes smoother, but less reactive to new samples. - */ -const constexpr char* ALPHA = "ALPHA"; -constexpr double_t DEFAULT_ALPHA = 0.2; - -const constexpr char* ALPHA_CPU = "ALPHA_CPU"; -const constexpr char* ALPHA_IPC = "ALPHA_IPC"; - -} // namespace ema - -namespace detector { -const constexpr char* ANALYZER_TYPE = "ANALYZER_TYPE"; -const constexpr char* WINDOW_SIZE = "WINDOW_SIZE"; -constexpr uint64_t DEFAULT_WINDOW_SIZE = 10; -const constexpr char* FRACTIONAL_THRESHOLD = "FRACTIONAL_THRESHOLD"; -constexpr double_t DEFAULT_FRACTIONAL_THRESHOLD = 0.5; -const constexpr char* SEVERITY_FRACTION = "SEVERITY_FRACTION"; -constexpr double_t DEFAULT_SEVERITY_FRACTION = -1; -const constexpr char* NEAR_FRACTION = "NEAR_FRACTION"; -constexpr double_t DEFAULT_NEAR_FRACTION = 0.1; -const constexpr char* MAX_CHECKPOINTS = "MAX_CHECKPOINTS"; -constexpr uint64_t DEFAULT_MAX_CHECKPOINTS = 3; -const constexpr char* QUORUM = "QUORUM"; -constexpr double_t DEFAULT_QUORUM = 0.70; - -constexpr double_t DEFAULT_START_VALUE = 0.00001; - -const constexpr char* THRESHOLD = "THRESHOLD"; -constexpr double_t DEFAULT_UTILIZATION_THRESHOLD = 0.85; -} // namespace detector - -namespace slack_observer { -constexpr double_t DEFAULT_MAX_OVERSUBSCRIPTION_FRACTION = 0.8; -} // namespace slack_observer - -namespace utilization { -constexpr double_t DEFAULT_THRESHOLD = 0.95; -} // namespace utilization - - -namespace new_executor { -constexpr uint32_t DEFAULT_THRESHOLD_SEC = 5 * 60; // !< Five minutes. -} // namespace new_executor - -namespace too_low_usage { -const constexpr char* MINIMAL_CPU_USAGE = "MINIMAL_CPU_USAGE"; -constexpr double_t DEFAULT_MINIMAL_CPU_USAGE = 0.25; // !< per sec. -} // namespace too_low_usage - -namespace strategy { -const constexpr char* CONTENTION_COOLDOWN = "CONTENTION_COOLDOWN"; -constexpr uint64_t DEFAULT_CONTENTION_COOLDOWN = 10; -const constexpr char* DEFAULT_CPU_SEVERITY = "DEFAULT_CPU_SEVERITY"; -constexpr double_t DEFAULT_DEFAULT_CPU_SEVERITY = 1.0; -static const constexpr char* STARTING_SEVERITY = "STARTING_SEVERITY"; -constexpr double_t DEFAULT_STARTING_SEVERITY = 0.1; -} // namespace strategy - -} // namespace serenity -} // namespace mesos - -#endif // SERENITY_DEFAULT_VARS_HPP diff --git a/src/serenity/resource_helper.cpp b/src/serenity/resource_helper.cpp index b105c6f..ce4f39a 100644 --- a/src/serenity/resource_helper.cpp +++ b/src/serenity/resource_helper.cpp @@ -43,36 +43,19 @@ ResourceUsageHelper::getProductionAndRevocableExecutors( revocableExecutors); } -Try ResourceUsageHelper::isProductionExecutor( +bool ResourceUsageHelper::isProductionExecutor( const ResourceUsage_Executor& executor) { - if (executor.allocated().size() == 0) { - return Error("Executor has no allocated resources."); - } - - if (Resources(executor.allocated()).revocable().empty()) { - return true; - } else { - return false; - } + return Resources(executor.allocated()).revocable().empty(); } bool ResourceUsageHelper::isExecutorHasStatistics( const ResourceUsage_Executor& executor) { - if (executor.has_executor_info() && executor.has_statistics()) { - return true; - } else { - return false; - } + return executor.has_statistics(); } -Try ResourceUsageHelper::isRevocableExecutor( +bool ResourceUsageHelper::isRevocableExecutor( const ResourceUsage_Executor &executor) { - Try result = isProductionExecutor(executor); - if (result.isError()) { - return result; - } - - return !result.get(); + return !isProductionExecutor(executor); } } // namespace serenity diff --git a/src/serenity/resource_helper.hpp b/src/serenity/resource_helper.hpp index bff7a9d..05823ae 100644 --- a/src/serenity/resource_helper.hpp +++ b/src/serenity/resource_helper.hpp @@ -34,18 +34,14 @@ class ResourceUsageHelper { getProductionAndRevocableExecutors(const ResourceUsage&); /** - * Checks if executor has empty revocable resources. - * - * Returns error when executor has no allocated resources. - */ - static Try isProductionExecutor(const ResourceUsage_Executor&); + * Checks if executor has empty revocable resources. + */ + static bool isProductionExecutor(const ResourceUsage_Executor&); /** - * Checks if executor has revocable resources. - * - * Returns error when executor has no allocated resources. - */ - static Try isRevocableExecutor(const ResourceUsage_Executor&); + * Checks if executor has revocable resources. + */ + static bool isRevocableExecutor(const ResourceUsage_Executor&); static bool isExecutorHasStatistics(const ResourceUsage_Executor&); diff --git a/src/serenity/serenity.hpp b/src/serenity/serenity.hpp index 01640b3..58b2089 100644 --- a/src/serenity/serenity.hpp +++ b/src/serenity/serenity.hpp @@ -8,6 +8,7 @@ #include "stout/nothing.hpp" #include "stout/try.hpp" +#include "stout/result.hpp" namespace mesos { namespace serenity { diff --git a/src/tests/common/config_helper.hpp b/src/tests/common/config_helper.hpp deleted file mode 100644 index 7070db9..0000000 --- a/src/tests/common/config_helper.hpp +++ /dev/null @@ -1,42 +0,0 @@ -#ifndef SERENITY_TESTS_CONFIG_HELPER_HPP -#define SERENITY_TESTS_CONFIG_HELPER_HPP - -#include - -#include "serenity/config.hpp" - -namespace mesos { -namespace serenity { -namespace tests { - -inline SerenityConfig createAssuranceAnalyzerCfg( - const uint64_t windowSize, - const uint64_t maxCheckpoints, - const double_t fractionalThreshold, - const double_t severityLvl = detector::DEFAULT_SEVERITY_FRACTION, - const double_t nearLvl = detector::DEFAULT_NEAR_FRACTION, - const double_t quorum = detector::DEFAULT_QUORUM) { - SerenityConfig cfg; - cfg.set(detector::WINDOW_SIZE, windowSize); - cfg.set(detector::MAX_CHECKPOINTS, maxCheckpoints); - cfg.set(detector::FRACTIONAL_THRESHOLD, fractionalThreshold); - cfg.set(detector::SEVERITY_FRACTION, severityLvl); - cfg.set(detector::NEAR_FRACTION, nearLvl); - cfg.set(detector::QUORUM, quorum); - - return cfg; -} - -inline SerenityConfig createThresholdDetectorCfg( - const double_t utilization = detector::DEFAULT_UTILIZATION_THRESHOLD) { - SerenityConfig cfg; - cfg.set(detector::THRESHOLD, utilization); - - return cfg; -} - -} // namespace tests -} // namespace serenity -} // namespace mesos - -#endif // SERENITY_TESTS_CONFIG_HELPER_HPP diff --git a/src/tests/contention_detectors/overload_test.cpp b/src/tests/contention_detectors/overload_test.cpp index 6b22a0e..edd7dfe 100644 --- a/src/tests/contention_detectors/overload_test.cpp +++ b/src/tests/contention_detectors/overload_test.cpp @@ -20,7 +20,6 @@ #include "stout/gtest.hpp" -#include "tests/common/config_helper.hpp" #include "tests/common/signal_helper.hpp" #include "tests/common/usage_helper.hpp" #include "tests/common/mocks/mock_sink.hpp" @@ -50,8 +49,9 @@ TEST(OverloadDetectorTest, LowUtilization) { OverloadDetector overloadDetector( &mockSink, usage::getCpuUsage, - createThresholdDetectorCfg( - UTIL_THRESHOLD)); + Config()); + + overloadDetector.setUtilizationThreshold(UTIL_THRESHOLD); // Fake slave ResourceUsage source. MockSource usageSource(&overloadDetector); @@ -101,8 +101,9 @@ TEST(OverloadDetectorTest, HighUtilization) { OverloadDetector overloadDetector( &mockSink, usage::getCpuUsage, - createThresholdDetectorCfg( - UTIL_THRESHOLD)); + Config()); + + overloadDetector.setUtilizationThreshold(UTIL_THRESHOLD); // Fake slave ResourceUsage source. MockSource usageSource(&overloadDetector); @@ -157,9 +158,9 @@ TEST(OverloadDetectorTest, IntegrationTest) { OverloadDetector overloadDetector( &mockSink, usage::getEmaCpuUsage, - createThresholdDetectorCfg( - UTIL_THRESHOLD)); + Config()); + overloadDetector.setUtilizationThreshold(UTIL_THRESHOLD); EMAFilter cpuEMAFilter(&overloadDetector, usage::getCpuUsage, diff --git a/src/tests/contention_detectors/signal_analyzers/drop_test.cpp b/src/tests/contention_detectors/signal_analyzers/drop_test.cpp index 1251fd6..8d5d493 100644 --- a/src/tests/contention_detectors/signal_analyzers/drop_test.cpp +++ b/src/tests/contention_detectors/signal_analyzers/drop_test.cpp @@ -11,8 +11,6 @@ #include "serenity/data_utils.hpp" -#include "tests/common/config_helper.hpp" - namespace mesos { namespace serenity { namespace tests { @@ -35,12 +33,14 @@ TEST(SignalDropAnalyzerTest, StableSignal) { SignalDropAnalyzer signalDropAnalyzer( Tag(QOS_CONTROLLER, "SignalDropAnalyzer"), - createAssuranceAnalyzerCfg( - WINDOWS_SIZE, - MAX_CHECKPOINTS, - FRACTION_THRESHOLD, - SEVERITY_FRACTION, - NEAR_FRACTION)); + Config()); + + signalDropAnalyzer.setWindowsSizeAndMaxCheckpoints( + WINDOWS_SIZE, + MAX_CHECKPOINTS); + signalDropAnalyzer.setFractionalThreshold(FRACTION_THRESHOLD); + signalDropAnalyzer.setSeverityFraction(SEVERITY_FRACTION); + signalDropAnalyzer.setNearFraction(NEAR_FRACTION); SignalScenario signalGen = SignalScenario(ITERATIONS) @@ -67,13 +67,15 @@ TEST(SignalDropAnalyzerTest, StableLoadOneBigDrop) { SignalDropAnalyzer signalDropAnalyzer( Tag(QOS_CONTROLLER, "SignalDropAnalyzer"), - createAssuranceAnalyzerCfg( - WINDOWS_SIZE, - MAX_CHECKPOINTS, - FRACTION_THRESHOLD, - SEVERITY_FRACTION, - NEAR_FRACTION, - QUORUM)); + Config()); + + signalDropAnalyzer.setWindowsSizeAndMaxCheckpoints( + WINDOWS_SIZE, + MAX_CHECKPOINTS); + signalDropAnalyzer.setFractionalThreshold(FRACTION_THRESHOLD); + signalDropAnalyzer.setSeverityFraction(SEVERITY_FRACTION); + signalDropAnalyzer.setNearFraction(NEAR_FRACTION); + signalDropAnalyzer.setQuroumFraction(QUORUM); SignalScenario signalGen = SignalScenario(ITERATIONS) @@ -95,23 +97,25 @@ TEST(SignalDropAnalyzerTest, StableLoadOneBigDrop) { TEST(SignalDropAnalyzerTest, StableLoadOneBigDropWithReset) { - const uint64_t WINDOWS_SIZE = 8; - const uint64_t MAX_CHECKPOINTS = 4; + const int64_t WINDOWS_SIZE = 8; + const int64_t MAX_CHECKPOINTS = 4; const double_t FRACTION_THRESHOLD = 0.5; const double_t SEVERITY_FRACTION = 1; const double_t NEAR_FRACTION = 0.1; const double_t QUORUM = 0.50; - const uint64_t ITERATIONS = 30; + const int64_t ITERATIONS = 30; SignalDropAnalyzer signalDropAnalyzer( Tag(QOS_CONTROLLER, "SignalDropAnalyzer"), - createAssuranceAnalyzerCfg( - WINDOWS_SIZE, - MAX_CHECKPOINTS, - FRACTION_THRESHOLD, - SEVERITY_FRACTION, - NEAR_FRACTION, - QUORUM)); + Config()); + + signalDropAnalyzer.setWindowsSizeAndMaxCheckpoints( + WINDOWS_SIZE, + MAX_CHECKPOINTS); + signalDropAnalyzer.setFractionalThreshold(FRACTION_THRESHOLD); + signalDropAnalyzer.setSeverityFraction(SEVERITY_FRACTION); + signalDropAnalyzer.setNearFraction(NEAR_FRACTION); + signalDropAnalyzer.setQuroumFraction(QUORUM); SignalScenario signalGen = SignalScenario(ITERATIONS) @@ -146,13 +150,15 @@ TEST(SignalDropAnalyzerTest, StableLoadOneProgressiveDrop) { SignalDropAnalyzer signalDropAnalyzer( Tag(QOS_CONTROLLER, "SignalDropAnalyzer"), - createAssuranceAnalyzerCfg( - WINDOWS_SIZE, - MAX_CHECKPOINTS, - FRACTION_THRESHOLD, - SEVERITY_FRACTION, - NEAR_FRACTION, - QUORUM)); + Config()); + + signalDropAnalyzer.setWindowsSizeAndMaxCheckpoints( + WINDOWS_SIZE, + MAX_CHECKPOINTS); + signalDropAnalyzer.setFractionalThreshold(FRACTION_THRESHOLD); + signalDropAnalyzer.setSeverityFraction(SEVERITY_FRACTION); + signalDropAnalyzer.setNearFraction(NEAR_FRACTION); + signalDropAnalyzer.setQuroumFraction(QUORUM); SignalScenario signalGen = SignalScenario(ITERATIONS) @@ -184,13 +190,15 @@ TEST(SignalDropAnalyzerTest, StableLoadOneBigDropAndRecovery) { SignalDropAnalyzer signalDropAnalyzer( Tag(QOS_CONTROLLER, "SignalDropAnalyzer"), - createAssuranceAnalyzerCfg( - WINDOWS_SIZE, - MAX_CHECKPOINTS, - FRACTION_THRESHOLD, - SEVERITY_FRACTION, - NEAR_FRACTION, - QUORUM)); + Config()); + + signalDropAnalyzer.setWindowsSizeAndMaxCheckpoints( + WINDOWS_SIZE, + MAX_CHECKPOINTS); + signalDropAnalyzer.setFractionalThreshold(FRACTION_THRESHOLD); + signalDropAnalyzer.setSeverityFraction(SEVERITY_FRACTION); + signalDropAnalyzer.setNearFraction(NEAR_FRACTION); + signalDropAnalyzer.setQuroumFraction(QUORUM); SignalScenario signalGen = SignalScenario(ITERATIONS) @@ -225,13 +233,15 @@ TEST(SignalDropAnalyzerTest, NoisyLoadOneBigDropLessCheckpoints) { SignalDropAnalyzer signalDropAnalyzer( Tag(QOS_CONTROLLER, "SignalDropAnalyzer"), - createAssuranceAnalyzerCfg( - WINDOWS_SIZE, - MAX_CHECKPOINTS, - FRACTION_THRESHOLD, - SEVERITY_FRACTION, - NEAR_FRACTION, - QUORUM)); + Config()); + + signalDropAnalyzer.setWindowsSizeAndMaxCheckpoints( + WINDOWS_SIZE, + MAX_CHECKPOINTS); + signalDropAnalyzer.setFractionalThreshold(FRACTION_THRESHOLD); + signalDropAnalyzer.setSeverityFraction(SEVERITY_FRACTION); + signalDropAnalyzer.setNearFraction(NEAR_FRACTION); + signalDropAnalyzer.setQuroumFraction(QUORUM); SignalScenario signalGen = SignalScenario(ITERATIONS) @@ -265,13 +275,15 @@ TEST(SignalDropAnalyzerTest, NoisyLoadOneBigDropMoreCheckpoints) { SignalDropAnalyzer signalDropAnalyzer( Tag(QOS_CONTROLLER, "SignalDropAnalyzer"), - createAssuranceAnalyzerCfg( - WINDOWS_SIZE, - MAX_CHECKPOINTS, - FRACTION_THRESHOLD, - SEVERITY_FRACTION, - NEAR_FRACTION, - QUORUM)); + Config()); + + signalDropAnalyzer.setWindowsSizeAndMaxCheckpoints( + WINDOWS_SIZE, + MAX_CHECKPOINTS); + signalDropAnalyzer.setFractionalThreshold(FRACTION_THRESHOLD); + signalDropAnalyzer.setSeverityFraction(SEVERITY_FRACTION); + signalDropAnalyzer.setNearFraction(NEAR_FRACTION); + signalDropAnalyzer.setQuroumFraction(QUORUM); SignalScenario signalGen = SignalScenario(ITERATIONS) diff --git a/src/tests/mesos_modules/qos_controller/qos_controller_test.cpp b/src/tests/mesos_modules/qos_controller/qos_controller_test.cpp index 1109df7..6b737cc 100644 --- a/src/tests/mesos_modules/qos_controller/qos_controller_test.cpp +++ b/src/tests/mesos_modules/qos_controller/qos_controller_test.cpp @@ -51,7 +51,7 @@ class TestCorrectionPipeline : public QoSControllerPipeline { TEST(SerenityControllerTest, PipelineIntegration) { Try qoSController = serenity::SerenityController::create( - std::shared_ptr( + std::unique_ptr( new TestCorrectionPipeline())); ASSERT_SOME(qoSController); diff --git a/src/tests/observers/strategies/seniority_strategy_test.cpp b/src/tests/observers/strategies/seniority_strategy_test.cpp index bca7ab5..0820397 100644 --- a/src/tests/observers/strategies/seniority_strategy_test.cpp +++ b/src/tests/observers/strategies/seniority_strategy_test.cpp @@ -50,7 +50,7 @@ TEST(QoSCorrectionObserverSeniorityDeciderTest, EmptyContentions) { ExecutorAgeFilter age; QoSCorrectionObserver observer( - &mockSink, &age, new SeniorityStrategy(SerenityConfig())); + &mockSink, &age, new SeniorityStrategy(Config())); age.addConsumer(&observer); @@ -104,7 +104,7 @@ TEST(QoSCorrectionObserverSeniorityDeciderTest, OneContentionSmallSeverity) { ExecutorAgeFilter age; QoSCorrectionObserver observer( - &mockSink, &age, new SeniorityStrategy(SerenityConfig())); + &mockSink, &age, new SeniorityStrategy(Config())); age.addConsumer(&observer); diff --git a/src/tests/serenity/config_test.cpp b/src/tests/serenity/config_test.cpp index 923579a..cddc424 100644 --- a/src/tests/serenity/config_test.cpp +++ b/src/tests/serenity/config_test.cpp @@ -5,13 +5,11 @@ #include "stout/gtest.hpp" -#include "tests/common/config_helper.hpp" - namespace mesos { namespace serenity { namespace tests { -// TestConfig required fields & default values using different types. +// TestConfig required items & default values using different types. const constexpr char* FIELD_STR = "FIELD_STR"; const constexpr char* DEFAULT_FIELD_STR = "default"; const constexpr char* MODIFIED_FIELD_STR = "modified"; @@ -20,10 +18,6 @@ const constexpr char* FIELD_BOOL = "FIELD_BOOL"; const constexpr bool DEFAULT_FIELD_BOOL = true; const constexpr bool MODIFIED_FIELD_BOOL = false; -const constexpr char* FIELD_UINT = "FIELD_UINT"; -const constexpr uint64_t DEFAULT_FIELD_UINT = 23424; -const constexpr uint64_t MODIFIED_FIELD_UINT = 3; - const constexpr char* FIELD_INT = "FIELD_INT"; const constexpr int64_t DEFAULT_FIELD_INT = -435; const constexpr int64_t MODIFIED_FIELD_INT = 34535; @@ -32,66 +26,147 @@ const constexpr char* FIELD_DOUBLE = "FIELD_DOUBLE"; const constexpr double_t DEFAULT_FIELD_DOUBLE = 0.345345; const constexpr double_t MODIFIED_FIELD_DOUBLE = 3.432; - -class TestConfig : public SerenityConfig { +class TestConfig : Config { public: - TestConfig() { - this->initDefaults(); + void loadSampleConfig() { + put(FIELD_STR, (std::string) MODIFIED_FIELD_STR); + put(FIELD_BOOL, MODIFIED_FIELD_BOOL); + put(FIELD_INT, MODIFIED_FIELD_INT); + put(FIELD_DOUBLE, MODIFIED_FIELD_DOUBLE); + } + + void loadSampleConfigWithSections(std::string sectionName) { + TestConfig config; + config.put(FIELD_STR, (std::string) MODIFIED_FIELD_STR); + config.put(FIELD_BOOL, MODIFIED_FIELD_BOOL); + config.put(FIELD_INT, MODIFIED_FIELD_INT); + config.put(FIELD_DOUBLE, MODIFIED_FIELD_DOUBLE); + + Config& config2 = getSectionRefOrNew(sectionName); + config2.applyConfig(config); } /** - * This constructor enables run-time overlapping of default - * configuration records. + * Put config value for Item types. */ - explicit TestConfig(const SerenityConfig& customCfg) { - this->initDefaults(); - this->applyConfig(customCfg); + template + void put(const std::string& key, T value) { + Config::putVariant(key, value); } /** - * Init default values for Test configuration. + * Getter for value in config. */ - void initDefaults() { - this->set(FIELD_STR, (std::string)DEFAULT_FIELD_STR); - this->set(FIELD_BOOL, DEFAULT_FIELD_BOOL); - this->set(FIELD_UINT, DEFAULT_FIELD_UINT); - this->set(FIELD_INT, DEFAULT_FIELD_INT); - this->set(FIELD_DOUBLE, DEFAULT_FIELD_DOUBLE); + template + const Result getValue(const std::string& key) const { + return Config::getValue(key); + } + + const Config& getSectionOrNew(const std::string& sectionKey) { + return Config::getSectionOrNew(sectionKey); } }; +TEST(ConfigTest, EmptyItemsTest) { + Config config; + EXPECT_NONE(config.getValue(FIELD_STR)); + EXPECT_NONE(config.getValue(FIELD_BOOL)); + EXPECT_NONE(config.getValue(FIELD_INT)); + EXPECT_NONE(config.getValue(FIELD_DOUBLE)); +} + +TEST(ConfigTest, DefaultItemsTest) { + Config config; + EXPECT_EQ(ConfigValidator( + config.getValue(FIELD_STR)).getOrElse(DEFAULT_FIELD_STR), + DEFAULT_FIELD_STR); + EXPECT_EQ(ConfigValidator( + config.getValue(FIELD_BOOL)).getOrElse(DEFAULT_FIELD_BOOL), + DEFAULT_FIELD_BOOL); + EXPECT_EQ(ConfigValidator( + config.getValue(FIELD_INT)).getOrElse(DEFAULT_FIELD_INT), + DEFAULT_FIELD_INT); + EXPECT_EQ(ConfigValidator( + config.getValue(FIELD_DOUBLE)).getOrElse(DEFAULT_FIELD_DOUBLE), + DEFAULT_FIELD_DOUBLE); +} -TEST(SerenityConfigTest, DefaultValuesAvailable) { - // Create empty config with no configuration fields. - SerenityConfig newConfig; - TestConfig internalConfig = TestConfig(newConfig); - EXPECT_EQ(internalConfig.getS(FIELD_STR), (std::string)DEFAULT_FIELD_STR); - EXPECT_EQ(internalConfig.getB(FIELD_BOOL), DEFAULT_FIELD_BOOL); - EXPECT_EQ(internalConfig.getU64(FIELD_UINT), DEFAULT_FIELD_UINT); - EXPECT_EQ(internalConfig.getI64(FIELD_INT), DEFAULT_FIELD_INT); - EXPECT_EQ(internalConfig.getD(FIELD_DOUBLE), DEFAULT_FIELD_DOUBLE); +TEST(ConfigTest, ModifiedItemsTest) { + TestConfig config; + EXPECT_EQ(ConfigValidator( + config.getValue(FIELD_STR)).getOrElse(DEFAULT_FIELD_STR), + DEFAULT_FIELD_STR); + EXPECT_EQ(ConfigValidator( + config.getValue(FIELD_BOOL)).getOrElse(DEFAULT_FIELD_BOOL), + DEFAULT_FIELD_BOOL); + EXPECT_EQ(ConfigValidator( + config.getValue(FIELD_INT)).getOrElse(DEFAULT_FIELD_INT), + DEFAULT_FIELD_INT); + EXPECT_EQ(ConfigValidator( + config.getValue(FIELD_DOUBLE)).getOrElse(DEFAULT_FIELD_DOUBLE), + DEFAULT_FIELD_DOUBLE); + + config.loadSampleConfig(); + + EXPECT_EQ(ConfigValidator( + config.getValue(FIELD_STR)).getOrElse(DEFAULT_FIELD_STR), + MODIFIED_FIELD_STR); + EXPECT_EQ(ConfigValidator( + config.getValue(FIELD_BOOL)).getOrElse(DEFAULT_FIELD_BOOL), + MODIFIED_FIELD_BOOL); + EXPECT_EQ(ConfigValidator( + config.getValue(FIELD_INT)).getOrElse(DEFAULT_FIELD_INT), + MODIFIED_FIELD_INT); + EXPECT_EQ(ConfigValidator( + config.getValue(FIELD_DOUBLE)).getOrElse(DEFAULT_FIELD_DOUBLE), + MODIFIED_FIELD_DOUBLE); } +TEST(ConfigTest, ErrorItemsTest) { + TestConfig config; + config.loadSampleConfig(); + EXPECT_ERROR(config.getValue(FIELD_STR)); + EXPECT_ERROR(config.getValue(FIELD_BOOL)); + EXPECT_ERROR(config.getValue(FIELD_INT)); + EXPECT_ERROR(config.getValue(FIELD_DOUBLE)); +} -TEST(SerenityConfigTest, ModifiedValuesAvailable) { - // Create config with custom configuration fields. - SerenityConfig newConfig; - newConfig.set(FIELD_STR, (std::string)MODIFIED_FIELD_STR); - newConfig.set(FIELD_BOOL, MODIFIED_FIELD_BOOL); - newConfig.set(FIELD_UINT, MODIFIED_FIELD_UINT); - newConfig.set(FIELD_INT, MODIFIED_FIELD_INT); - newConfig.set(FIELD_DOUBLE, MODIFIED_FIELD_DOUBLE); - - SerenityConfig internalConfig = TestConfig(newConfig); - EXPECT_EQ(internalConfig.getS(FIELD_STR), (std::string)MODIFIED_FIELD_STR); - EXPECT_EQ(internalConfig.getB(FIELD_BOOL), MODIFIED_FIELD_BOOL); - EXPECT_EQ(internalConfig.getU64(FIELD_UINT), MODIFIED_FIELD_UINT); - EXPECT_EQ(internalConfig.getI64(FIELD_INT), MODIFIED_FIELD_INT); - EXPECT_EQ(internalConfig.getD(FIELD_DOUBLE), MODIFIED_FIELD_DOUBLE); +TEST(ConfigTest, ModifiedSectionItemsTest) { + const constexpr char* FIELD_SECTION = "SECTION1"; + TestConfig config; + const Config& section = config.getSectionOrNew(FIELD_SECTION); + + EXPECT_EQ(ConfigValidator( + section.getValue(FIELD_STR)).getOrElse(DEFAULT_FIELD_STR), + DEFAULT_FIELD_STR); + EXPECT_EQ(ConfigValidator( + section.getValue(FIELD_BOOL)).getOrElse(DEFAULT_FIELD_BOOL), + DEFAULT_FIELD_BOOL); + EXPECT_EQ(ConfigValidator( + section.getValue(FIELD_INT)).getOrElse(DEFAULT_FIELD_INT), + DEFAULT_FIELD_INT); + EXPECT_EQ(ConfigValidator( + section.getValue(FIELD_DOUBLE)).getOrElse(DEFAULT_FIELD_DOUBLE), + DEFAULT_FIELD_DOUBLE); + + config.loadSampleConfigWithSections(FIELD_SECTION); + + EXPECT_EQ(ConfigValidator( + section.getValue(FIELD_STR)).getOrElse(DEFAULT_FIELD_STR), + MODIFIED_FIELD_STR); + EXPECT_EQ(ConfigValidator( + section.getValue(FIELD_BOOL)).getOrElse(DEFAULT_FIELD_BOOL), + MODIFIED_FIELD_BOOL); + EXPECT_EQ(ConfigValidator( + section.getValue(FIELD_INT)).getOrElse(DEFAULT_FIELD_INT), + MODIFIED_FIELD_INT); + EXPECT_EQ(ConfigValidator( + section.getValue(FIELD_DOUBLE)).getOrElse(DEFAULT_FIELD_DOUBLE), + MODIFIED_FIELD_DOUBLE); } + } // namespace tests } // namespace serenity } // namespace mesos -