From 6ec8a9251a4b980524618c7248e92cbdbb7ac683 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Reni=C3=A9?= Date: Tue, 23 Oct 2018 14:19:23 +0200 Subject: [PATCH] project: add black formatter --- docs/conf.py | 1 - graphite_api/_vendor/whisper.py | 1621 ++++++++-------- graphite_api/app.py | 509 ++--- graphite_api/carbonlink.py | 141 +- graphite_api/config.py | 230 ++- graphite_api/encoders.py | 7 +- graphite_api/evaluator.py | 121 +- graphite_api/finders/__init__.py | 26 +- graphite_api/finders/whisper.py | 112 +- graphite_api/functions.py | 1371 ++++++++------ graphite_api/intervals.py | 39 +- graphite_api/middleware.py | 21 +- graphite_api/node.py | 14 +- graphite_api/readers.py | 7 +- graphite_api/render/attime.py | 142 +- graphite_api/render/datalib.py | 98 +- graphite_api/render/glyph.py | 2034 +++++++++++--------- graphite_api/render/grammar.py | 142 +- graphite_api/storage.py | 17 +- graphite_api/utils.py | 20 +- setup.cfg | 1 + setup.py | 87 +- tests/__init__.py | 37 +- tests/test_attime.py | 80 +- tests/test_carbonlink.py | 108 +- tests/test_finders.py | 113 +- tests/test_functions.py | 2960 +++++++++++++++++++++--------- tests/test_http.py | 42 +- tests/test_intervals.py | 22 +- tests/test_metrics.py | 482 ++--- tests/test_paths.py | 25 +- tests/test_render.py | 1017 ++++++---- tests/test_render_datalib.py | 148 +- tests/test_render_glyph.py | 586 +++--- tests/test_storage.py | 20 +- tox.ini | 4 +- unittest_main.py | 6 +- 37 files changed, 7476 insertions(+), 4935 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index b38144c..7486138 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -6,7 +6,6 @@ import sys import sphinx_rtd_theme - from sphinx.ext import autodoc sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir)) diff --git a/graphite_api/_vendor/whisper.py b/graphite_api/_vendor/whisper.py index 95e5830..ebb1e0c 100644 --- a/graphite_api/_vendor/whisper.py +++ b/graphite_api/_vendor/whisper.py @@ -18,12 +18,12 @@ # Here is the basic layout of a whisper data file # # File = Header,Data -# Header = Metadata,ArchiveInfo+ -# Metadata = aggregationType,maxRetention,xFilesFactor,archiveCount -# ArchiveInfo = Offset,SecondsPerPoint,Points -# Data = Archive+ -# Archive = Point+ -# Point = timestamp,value +# Header = Metadata,ArchiveInfo+ +# Metadata = aggregationType,maxRetention,xFilesFactor,archiveCount +# ArchiveInfo = Offset,SecondsPerPoint,Points +# Data = Archive+ +# Archive = Point+ +# Point = timestamp,value import itertools import operator @@ -32,53 +32,57 @@ import sys import time -izip = getattr(itertools, 'izip', zip) -ifilter = getattr(itertools, 'ifilter', filter) +izip = getattr(itertools, "izip", zip) +ifilter = getattr(itertools, "ifilter", filter) if sys.version_info >= (3, 0): xrange = range try: - import fcntl - CAN_LOCK = True + import fcntl + + CAN_LOCK = True except ImportError: - CAN_LOCK = False + CAN_LOCK = False try: - import ctypes - import ctypes.util - CAN_FALLOCATE = True + import ctypes + import ctypes.util + + CAN_FALLOCATE = True except ImportError: - CAN_FALLOCATE = False + CAN_FALLOCATE = False fallocate = None if CAN_FALLOCATE: - libc_name = ctypes.util.find_library('c') - libc = ctypes.CDLL(libc_name) - c_off64_t = ctypes.c_int64 - c_off_t = ctypes.c_int - - try: - _fallocate = libc.posix_fallocate64 - _fallocate.restype = ctypes.c_int - _fallocate.argtypes = [ctypes.c_int, c_off64_t, c_off64_t] - except AttributeError: + libc_name = ctypes.util.find_library("c") + libc = ctypes.CDLL(libc_name) + c_off64_t = ctypes.c_int64 + c_off_t = ctypes.c_int + try: - _fallocate = libc.posix_fallocate - _fallocate.restype = ctypes.c_int - _fallocate.argtypes = [ctypes.c_int, c_off_t, c_off_t] + _fallocate = libc.posix_fallocate64 + _fallocate.restype = ctypes.c_int + _fallocate.argtypes = [ctypes.c_int, c_off64_t, c_off64_t] except AttributeError: - CAN_FALLOCATE = False + try: + _fallocate = libc.posix_fallocate + _fallocate.restype = ctypes.c_int + _fallocate.argtypes = [ctypes.c_int, c_off_t, c_off_t] + except AttributeError: + CAN_FALLOCATE = False + + if CAN_FALLOCATE: - if CAN_FALLOCATE: - def _py_fallocate(fd, offset, len_): - res = _fallocate(fd.fileno(), offset, len_) - if res != 0: - raise IOError(res, 'fallocate') - fallocate = _py_fallocate - del libc - del libc_name + def _py_fallocate(fd, offset, len_): + res = _fallocate(fd.fileno(), offset, len_) + if res != 0: + raise IOError(res, "fallocate") + + fallocate = _py_fallocate + del libc + del libc_name LOCK = False CACHE_HEADERS = False @@ -98,62 +102,77 @@ def _py_fallocate(fd, offset, len_): archiveInfoFormat = "!3L" archiveInfoSize = struct.calcsize(archiveInfoFormat) -aggregationTypeToMethod = dict({ - 1: 'average', - 2: 'sum', - 3: 'last', - 4: 'max', - 5: 'min' -}) -aggregationMethodToType = dict([[v,k] for k,v in aggregationTypeToMethod.items()]) +aggregationTypeToMethod = dict( + {1: "average", 2: "sum", 3: "last", 4: "max", 5: "min"} +) +aggregationMethodToType = dict( + [[v, k] for k, v in aggregationTypeToMethod.items()] +) aggregationMethods = aggregationTypeToMethod.values() -debug = startBlock = endBlock = lambda *a,**k: None +debug = startBlock = endBlock = lambda *a, **k: None UnitMultipliers = { - 'seconds' : 1, - 'minutes' : 60, - 'hours' : 3600, - 'days' : 86400, - 'weeks' : 86400 * 7, - 'years' : 86400 * 365 + "seconds": 1, + "minutes": 60, + "hours": 3600, + "days": 86400, + "weeks": 86400 * 7, + "years": 86400 * 365, } def getUnitString(s): - if 'seconds'.startswith(s): return 'seconds' - if 'minutes'.startswith(s): return 'minutes' - if 'hours'.startswith(s): return 'hours' - if 'days'.startswith(s): return 'days' - if 'weeks'.startswith(s): return 'weeks' - if 'years'.startswith(s): return 'years' - raise ValueError("Invalid unit '%s'" % s) + if "seconds".startswith(s): + return "seconds" + if "minutes".startswith(s): + return "minutes" + if "hours".startswith(s): + return "hours" + if "days".startswith(s): + return "days" + if "weeks".startswith(s): + return "weeks" + if "years".startswith(s): + return "years" + raise ValueError("Invalid unit '%s'" % s) + def parseRetentionDef(retentionDef): - import re - (precision, points) = retentionDef.strip().split(':') - - if precision.isdigit(): - precision = int(precision) * UnitMultipliers[getUnitString('s')] - else: - precision_re = re.compile(r'^(\d+)([a-z]+)$') - match = precision_re.match(precision) - if match: - precision = int(match.group(1)) * UnitMultipliers[getUnitString(match.group(2))] + import re + + (precision, points) = retentionDef.strip().split(":") + + if precision.isdigit(): + precision = int(precision) * UnitMultipliers[getUnitString("s")] else: - raise ValueError("Invalid precision specification '%s'" % precision) - - if points.isdigit(): - points = int(points) - else: - points_re = re.compile(r'^(\d+)([a-z]+)$') - match = points_re.match(points) - if match: - points = int(match.group(1)) * UnitMultipliers[getUnitString(match.group(2))] // precision + precision_re = re.compile(r"^(\d+)([a-z]+)$") + match = precision_re.match(precision) + if match: + precision = ( + int(match.group(1)) + * UnitMultipliers[getUnitString(match.group(2))] + ) + else: + raise ValueError( + "Invalid precision specification '%s'" % precision + ) + + if points.isdigit(): + points = int(points) else: - raise ValueError("Invalid retention specification '%s'" % points) + points_re = re.compile(r"^(\d+)([a-z]+)$") + match = points_re.match(points) + if match: + points = ( + int(match.group(1)) + * UnitMultipliers[getUnitString(match.group(2))] + // precision + ) + else: + raise ValueError("Invalid retention specification '%s'" % points) - return (precision, points) + return (precision, points) class WhisperException(Exception): @@ -175,152 +194,176 @@ class InvalidTimeInterval(WhisperException): class TimestampNotCovered(WhisperException): """Timestamp not covered by any archives in this database.""" + class CorruptWhisperFile(WhisperException): - def __init__(self, error, path): - Exception.__init__(self, error) - self.error = error - self.path = path + def __init__(self, error, path): + Exception.__init__(self, error) + self.error = error + self.path = path - def __repr__(self): - return "" % (self.path, self.error) + def __repr__(self): + return "" % (self.path, self.error) - def __str__(self): - return "%s (%s)" % (self.error, self.path) + def __str__(self): + return "%s (%s)" % (self.error, self.path) -def enableDebug(): - global open, debug, startBlock, endBlock - class open(file): - def __init__(self,*args,**kwargs): - file.__init__(self,*args,**kwargs) - self.writeCount = 0 - self.readCount = 0 - def write(self,data): - self.writeCount += 1 - debug('WRITE %d bytes #%d' % (len(data),self.writeCount)) - return file.write(self,data) +def enableDebug(): + global open, debug, startBlock, endBlock - def read(self,bytes): - self.readCount += 1 - debug('READ %d bytes #%d' % (bytes,self.readCount)) - return file.read(self,bytes) + class open(file): + def __init__(self, *args, **kwargs): + file.__init__(self, *args, **kwargs) + self.writeCount = 0 + self.readCount = 0 - def debug(message): - print('DEBUG :: %s' % message) + def write(self, data): + self.writeCount += 1 + debug("WRITE %d bytes #%d" % (len(data), self.writeCount)) + return file.write(self, data) - __timingBlocks = {} + def read(self, bytes): + self.readCount += 1 + debug("READ %d bytes #%d" % (bytes, self.readCount)) + return file.read(self, bytes) - def startBlock(name): - __timingBlocks[name] = time.time() + def debug(message): + print("DEBUG :: %s" % message) - def endBlock(name): - debug("%s took %.5f seconds" % (name,time.time() - __timingBlocks.pop(name))) + __timingBlocks = {} + def startBlock(name): + __timingBlocks[name] = time.time() -def __readHeader(fh): - info = __headerCache.get(fh.name) - if info: - return info + def endBlock(name): + debug( + "%s took %.5f seconds" + % (name, time.time() - __timingBlocks.pop(name)) + ) - originalOffset = fh.tell() - fh.seek(0) - packedMetadata = fh.read(metadataSize) - try: - (aggregationType,maxRetention,xff,archiveCount) = struct.unpack(metadataFormat,packedMetadata) - except: - raise CorruptWhisperFile("Unable to read header", fh.name) +def __readHeader(fh): + info = __headerCache.get(fh.name) + if info: + return info - archives = [] + originalOffset = fh.tell() + fh.seek(0) + packedMetadata = fh.read(metadataSize) - for i in xrange(archiveCount): - packedArchiveInfo = fh.read(archiveInfoSize) try: - (offset,secondsPerPoint,points) = struct.unpack(archiveInfoFormat,packedArchiveInfo) + (aggregationType, maxRetention, xff, archiveCount) = struct.unpack( + metadataFormat, packedMetadata + ) except: - raise CorruptWhisperFile("Unable to read archive%d metadata" % i, fh.name) - - archiveInfo = { - 'offset' : offset, - 'secondsPerPoint' : secondsPerPoint, - 'points' : points, - 'retention' : secondsPerPoint * points, - 'size' : points * pointSize, + raise CorruptWhisperFile("Unable to read header", fh.name) + + archives = [] + + for i in xrange(archiveCount): + packedArchiveInfo = fh.read(archiveInfoSize) + try: + (offset, secondsPerPoint, points) = struct.unpack( + archiveInfoFormat, packedArchiveInfo + ) + except: + raise CorruptWhisperFile( + "Unable to read archive%d metadata" % i, fh.name + ) + + archiveInfo = { + "offset": offset, + "secondsPerPoint": secondsPerPoint, + "points": points, + "retention": secondsPerPoint * points, + "size": points * pointSize, + } + archives.append(archiveInfo) + + fh.seek(originalOffset) + info = { + "aggregationMethod": aggregationTypeToMethod.get( + aggregationType, "average" + ), + "maxRetention": maxRetention, + "xFilesFactor": xff, + "archives": archives, } - archives.append(archiveInfo) - - fh.seek(originalOffset) - info = { - 'aggregationMethod' : aggregationTypeToMethod.get(aggregationType, 'average'), - 'maxRetention' : maxRetention, - 'xFilesFactor' : xff, - 'archives' : archives, - } - if CACHE_HEADERS: - __headerCache[fh.name] = info + if CACHE_HEADERS: + __headerCache[fh.name] = info - return info + return info def setAggregationMethod(path, aggregationMethod, xFilesFactor=None): - """setAggregationMethod(path,aggregationMethod,xFilesFactor=None) + """setAggregationMethod(path,aggregationMethod,xFilesFactor=None) path is a string aggregationMethod specifies the method to use when propagating data (see ``whisper.aggregationMethods``) xFilesFactor specifies the fraction of data points in a propagation interval that must have known values for a propagation to occur. If None, the existing xFilesFactor in path will not be changed """ - fh = None - try: - - fh = open(path,'r+b') - if LOCK: - fcntl.flock( fh.fileno(), fcntl.LOCK_EX ) - - packedMetadata = fh.read(metadataSize) - + fh = None try: - (aggregationType,maxRetention,xff,archiveCount) = struct.unpack(metadataFormat,packedMetadata) - except: - raise CorruptWhisperFile("Unable to read header", fh.name) - try: - newAggregationType = struct.pack( longFormat, aggregationMethodToType[aggregationMethod] ) - except KeyError: - raise InvalidAggregationMethod("Unrecognized aggregation method: %s" % - aggregationMethod) - - if xFilesFactor is not None: - #use specified xFilesFactor - xff = struct.pack( floatFormat, float(xFilesFactor) ) - else: - #retain old value - xff = struct.pack( floatFormat, xff ) - - #repack the remaining header information - maxRetention = struct.pack( longFormat, maxRetention ) - archiveCount = struct.pack(longFormat, archiveCount) - - packedMetadata = newAggregationType + maxRetention + xff + archiveCount - fh.seek(0) - #fh.write(newAggregationType) - fh.write(packedMetadata) - - if AUTOFLUSH: - fh.flush() - os.fsync(fh.fileno()) - - if CACHE_HEADERS and fh.name in __headerCache: - del __headerCache[fh.name] - - finally: - if fh: - fh.close() - - return aggregationTypeToMethod.get(aggregationType, 'average') + fh = open(path, "r+b") + if LOCK: + fcntl.flock(fh.fileno(), fcntl.LOCK_EX) + + packedMetadata = fh.read(metadataSize) + + try: + ( + aggregationType, + maxRetention, + xff, + archiveCount, + ) = struct.unpack(metadataFormat, packedMetadata) + except: + raise CorruptWhisperFile("Unable to read header", fh.name) + + try: + newAggregationType = struct.pack( + longFormat, aggregationMethodToType[aggregationMethod] + ) + except KeyError: + raise InvalidAggregationMethod( + "Unrecognized aggregation method: %s" % aggregationMethod + ) + + if xFilesFactor is not None: + # use specified xFilesFactor + xff = struct.pack(floatFormat, float(xFilesFactor)) + else: + # retain old value + xff = struct.pack(floatFormat, xff) + + # repack the remaining header information + maxRetention = struct.pack(longFormat, maxRetention) + archiveCount = struct.pack(longFormat, archiveCount) + + packedMetadata = ( + newAggregationType + maxRetention + xff + archiveCount + ) + fh.seek(0) + # fh.write(newAggregationType) + fh.write(packedMetadata) + + if AUTOFLUSH: + fh.flush() + os.fsync(fh.fileno()) + + if CACHE_HEADERS and fh.name in __headerCache: + del __headerCache[fh.name] + + finally: + if fh: + fh.close() + + return aggregationTypeToMethod.get(aggregationType, "average") def validateArchiveList(archiveList): - """ Validates an archiveList. + """ Validates an archiveList. An ArchiveList must: 1. Have at least one archive config. Example: (60, 86400) 2. No archive may be a duplicate of another. @@ -331,415 +374,490 @@ def validateArchiveList(archiveList): Returns True or False """ - if not archiveList: - raise InvalidConfiguration("You must specify at least one archive configuration!") - - archiveList = sorted(archiveList, key=lambda a: a[0]) #sort by precision (secondsPerPoint) - - for i,archive in enumerate(archiveList): - if i == len(archiveList) - 1: - break - - nextArchive = archiveList[i+1] - if not archive[0] < nextArchive[0]: - raise InvalidConfiguration("A Whisper database may not configured having" - "two archives with the same precision (archive%d: %s, archive%d: %s)" % - (i, archive, i + 1, nextArchive)) - - if nextArchive[0] % archive[0] != 0: - raise InvalidConfiguration("Higher precision archives' precision " - "must evenly divide all lower precision archives' precision " - "(archive%d: %s, archive%d: %s)" % - (i, archive[0], i + 1, nextArchive[0])) - - retention = archive[0] * archive[1] - nextRetention = nextArchive[0] * nextArchive[1] - - if not nextRetention > retention: - raise InvalidConfiguration("Lower precision archives must cover " - "larger time intervals than higher precision archives " - "(archive%d: %s seconds, archive%d: %s seconds)" % - (i, retention, i + 1, nextRetention)) - - archivePoints = archive[1] - pointsPerConsolidation = nextArchive[0] // archive[0] - if not archivePoints >= pointsPerConsolidation: - raise InvalidConfiguration("Each archive must have at least enough points " - "to consolidate to the next archive (archive%d consolidates %d of " - "archive%d's points but it has only %d total points)" % - (i + 1, pointsPerConsolidation, i, archivePoints)) - - -def create(path,archiveList,xFilesFactor=None,aggregationMethod=None,sparse=False,useFallocate=False): - """create(path,archiveList,xFilesFactor=0.5,aggregationMethod='average') + if not archiveList: + raise InvalidConfiguration( + "You must specify at least one archive configuration!" + ) + + archiveList = sorted( + archiveList, key=lambda a: a[0] + ) # sort by precision (secondsPerPoint) + + for i, archive in enumerate(archiveList): + if i == len(archiveList) - 1: + break + + nextArchive = archiveList[i + 1] + if not archive[0] < nextArchive[0]: + raise InvalidConfiguration( + "A Whisper database may not configured having" + "two archives with the same precision (archive%d: %s, archive%d: %s)" + % (i, archive, i + 1, nextArchive) + ) + + if nextArchive[0] % archive[0] != 0: + raise InvalidConfiguration( + "Higher precision archives' precision " + "must evenly divide all lower precision archives' precision " + "(archive%d: %s, archive%d: %s)" + % (i, archive[0], i + 1, nextArchive[0]) + ) + + retention = archive[0] * archive[1] + nextRetention = nextArchive[0] * nextArchive[1] + + if not nextRetention > retention: + raise InvalidConfiguration( + "Lower precision archives must cover " + "larger time intervals than higher precision archives " + "(archive%d: %s seconds, archive%d: %s seconds)" + % (i, retention, i + 1, nextRetention) + ) + + archivePoints = archive[1] + pointsPerConsolidation = nextArchive[0] // archive[0] + if not archivePoints >= pointsPerConsolidation: + raise InvalidConfiguration( + "Each archive must have at least enough points " + "to consolidate to the next archive (archive%d consolidates %d of " + "archive%d's points but it has only %d total points)" + % (i + 1, pointsPerConsolidation, i, archivePoints) + ) + + +def create( + path, + archiveList, + xFilesFactor=None, + aggregationMethod=None, + sparse=False, + useFallocate=False, +): + """create(path,archiveList,xFilesFactor=0.5,aggregationMethod='average') path is a string archiveList is a list of archives, each of which is of the form (secondsPerPoint,numberOfPoints) xFilesFactor specifies the fraction of data points in a propagation interval that must have known values for a propagation to occur aggregationMethod specifies the function to use when propagating data (see ``whisper.aggregationMethods``) """ - # Set default params - if xFilesFactor is None: - xFilesFactor = 0.5 - if aggregationMethod is None: - aggregationMethod = 'average' - - #Validate archive configurations... - validateArchiveList(archiveList) - - #Looks good, now we create the file and write the header - if os.path.exists(path): - raise InvalidConfiguration("File %s already exists!" % path) - fh = None - try: - fh = open(path,'wb') - if LOCK: - fcntl.flock( fh.fileno(), fcntl.LOCK_EX ) - - aggregationType = struct.pack( longFormat, aggregationMethodToType.get(aggregationMethod, 1) ) - oldest = max([secondsPerPoint * points for secondsPerPoint,points in archiveList]) - maxRetention = struct.pack( longFormat, oldest ) - xFilesFactor = struct.pack( floatFormat, float(xFilesFactor) ) - archiveCount = struct.pack(longFormat, len(archiveList)) - packedMetadata = aggregationType + maxRetention + xFilesFactor + archiveCount - fh.write(packedMetadata) - headerSize = metadataSize + (archiveInfoSize * len(archiveList)) - archiveOffsetPointer = headerSize - - for secondsPerPoint,points in archiveList: - archiveInfo = struct.pack(archiveInfoFormat, archiveOffsetPointer, secondsPerPoint, points) - fh.write(archiveInfo) - archiveOffsetPointer += (points * pointSize) - - #If configured to use fallocate and capable of fallocate use that, else - #attempt sparse if configure or zero pre-allocate if sparse isn't configured. - if CAN_FALLOCATE and useFallocate: - remaining = archiveOffsetPointer - headerSize - fallocate(fh, headerSize, remaining) - elif sparse: - fh.seek(archiveOffsetPointer - 1) - fh.write('\x00') - else: - remaining = archiveOffsetPointer - headerSize - chunksize = 16384 - zeroes = b'\x00' * chunksize - while remaining > chunksize: - fh.write(zeroes) - remaining -= chunksize - fh.write(zeroes[:remaining]) + # Set default params + if xFilesFactor is None: + xFilesFactor = 0.5 + if aggregationMethod is None: + aggregationMethod = "average" + + # Validate archive configurations... + validateArchiveList(archiveList) + + # Looks good, now we create the file and write the header + if os.path.exists(path): + raise InvalidConfiguration("File %s already exists!" % path) + fh = None + try: + fh = open(path, "wb") + if LOCK: + fcntl.flock(fh.fileno(), fcntl.LOCK_EX) + + aggregationType = struct.pack( + longFormat, aggregationMethodToType.get(aggregationMethod, 1) + ) + oldest = max( + [ + secondsPerPoint * points + for secondsPerPoint, points in archiveList + ] + ) + maxRetention = struct.pack(longFormat, oldest) + xFilesFactor = struct.pack(floatFormat, float(xFilesFactor)) + archiveCount = struct.pack(longFormat, len(archiveList)) + packedMetadata = ( + aggregationType + maxRetention + xFilesFactor + archiveCount + ) + fh.write(packedMetadata) + headerSize = metadataSize + (archiveInfoSize * len(archiveList)) + archiveOffsetPointer = headerSize + + for secondsPerPoint, points in archiveList: + archiveInfo = struct.pack( + archiveInfoFormat, + archiveOffsetPointer, + secondsPerPoint, + points, + ) + fh.write(archiveInfo) + archiveOffsetPointer += points * pointSize + + # If configured to use fallocate and capable of fallocate use that, else + # attempt sparse if configure or zero pre-allocate if sparse isn't configured. + if CAN_FALLOCATE and useFallocate: + remaining = archiveOffsetPointer - headerSize + fallocate(fh, headerSize, remaining) + elif sparse: + fh.seek(archiveOffsetPointer - 1) + fh.write("\x00") + else: + remaining = archiveOffsetPointer - headerSize + chunksize = 16384 + zeroes = b"\x00" * chunksize + while remaining > chunksize: + fh.write(zeroes) + remaining -= chunksize + fh.write(zeroes[:remaining]) + + if AUTOFLUSH: + fh.flush() + os.fsync(fh.fileno()) + finally: + if fh: + fh.close() - if AUTOFLUSH: - fh.flush() - os.fsync(fh.fileno()) - finally: - if fh: - fh.close() def aggregate(aggregationMethod, knownValues): - if aggregationMethod == 'average': - return float(sum(knownValues)) / float(len(knownValues)) - elif aggregationMethod == 'sum': - return float(sum(knownValues)) - elif aggregationMethod == 'last': - return knownValues[len(knownValues)-1] - elif aggregationMethod == 'max': - return max(knownValues) - elif aggregationMethod == 'min': - return min(knownValues) - else: - raise InvalidAggregationMethod("Unrecognized aggregation method %s" % - aggregationMethod) - - -def __propagate(fh,header,timestamp,higher,lower): - aggregationMethod = header['aggregationMethod'] - xff = header['xFilesFactor'] - - lowerIntervalStart = timestamp - (timestamp % lower['secondsPerPoint']) - lowerIntervalEnd = lowerIntervalStart + lower['secondsPerPoint'] - - fh.seek(higher['offset']) - packedPoint = fh.read(pointSize) - (higherBaseInterval,higherBaseValue) = struct.unpack(pointFormat,packedPoint) - - if higherBaseInterval == 0: - higherFirstOffset = higher['offset'] - else: - timeDistance = lowerIntervalStart - higherBaseInterval - pointDistance = timeDistance // higher['secondsPerPoint'] - byteDistance = pointDistance * pointSize - higherFirstOffset = higher['offset'] + (byteDistance % higher['size']) - - higherPoints = lower['secondsPerPoint'] // higher['secondsPerPoint'] - higherSize = higherPoints * pointSize - relativeFirstOffset = higherFirstOffset - higher['offset'] - relativeLastOffset = (relativeFirstOffset + higherSize) % higher['size'] - higherLastOffset = relativeLastOffset + higher['offset'] - fh.seek(higherFirstOffset) - - if higherFirstOffset < higherLastOffset: #we don't wrap the archive - seriesString = fh.read(higherLastOffset - higherFirstOffset) - else: #We do wrap the archive - higherEnd = higher['offset'] + higher['size'] - seriesString = fh.read(higherEnd - higherFirstOffset) - fh.seek(higher['offset']) - seriesString += fh.read(higherLastOffset - higher['offset']) - - #Now we unpack the series data we just read - byteOrder,pointTypes = pointFormat[0],pointFormat[1:] - points = len(seriesString) // pointSize - seriesFormat = byteOrder + (pointTypes * points) - unpackedSeries = struct.unpack(seriesFormat, seriesString) - - #And finally we construct a list of values - neighborValues = [None] * points - currentInterval = lowerIntervalStart - step = higher['secondsPerPoint'] - - for i in xrange(0,len(unpackedSeries),2): - pointTime = unpackedSeries[i] - if pointTime == currentInterval: - neighborValues[i//2] = unpackedSeries[i+1] - currentInterval += step - - #Propagate aggregateValue to propagate from neighborValues if we have enough known points - knownValues = [v for v in neighborValues if v is not None] - if not knownValues: - return False - - knownPercent = float(len(knownValues)) / float(len(neighborValues)) - if knownPercent >= xff: #we have enough data to propagate a value! - aggregateValue = aggregate(aggregationMethod, knownValues) - myPackedPoint = struct.pack(pointFormat,lowerIntervalStart,aggregateValue) - fh.seek(lower['offset']) - packedPoint = fh.read(pointSize) - (lowerBaseInterval,lowerBaseValue) = struct.unpack(pointFormat,packedPoint) + if aggregationMethod == "average": + return float(sum(knownValues)) / float(len(knownValues)) + elif aggregationMethod == "sum": + return float(sum(knownValues)) + elif aggregationMethod == "last": + return knownValues[len(knownValues) - 1] + elif aggregationMethod == "max": + return max(knownValues) + elif aggregationMethod == "min": + return min(knownValues) + else: + raise InvalidAggregationMethod( + "Unrecognized aggregation method %s" % aggregationMethod + ) - if lowerBaseInterval == 0: #First propagated update to this lower archive - fh.seek(lower['offset']) - fh.write(myPackedPoint) - else: #Not our first propagated update to this lower archive - timeDistance = lowerIntervalStart - lowerBaseInterval - pointDistance = timeDistance // lower['secondsPerPoint'] - byteDistance = pointDistance * pointSize - lowerOffset = lower['offset'] + (byteDistance % lower['size']) - fh.seek(lowerOffset) - fh.write(myPackedPoint) - return True +def __propagate(fh, header, timestamp, higher, lower): + aggregationMethod = header["aggregationMethod"] + xff = header["xFilesFactor"] - else: - return False + lowerIntervalStart = timestamp - (timestamp % lower["secondsPerPoint"]) + lowerIntervalEnd = lowerIntervalStart + lower["secondsPerPoint"] + fh.seek(higher["offset"]) + packedPoint = fh.read(pointSize) + (higherBaseInterval, higherBaseValue) = struct.unpack( + pointFormat, packedPoint + ) + + if higherBaseInterval == 0: + higherFirstOffset = higher["offset"] + else: + timeDistance = lowerIntervalStart - higherBaseInterval + pointDistance = timeDistance // higher["secondsPerPoint"] + byteDistance = pointDistance * pointSize + higherFirstOffset = higher["offset"] + (byteDistance % higher["size"]) + + higherPoints = lower["secondsPerPoint"] // higher["secondsPerPoint"] + higherSize = higherPoints * pointSize + relativeFirstOffset = higherFirstOffset - higher["offset"] + relativeLastOffset = (relativeFirstOffset + higherSize) % higher["size"] + higherLastOffset = relativeLastOffset + higher["offset"] + fh.seek(higherFirstOffset) + + if higherFirstOffset < higherLastOffset: # we don't wrap the archive + seriesString = fh.read(higherLastOffset - higherFirstOffset) + else: # We do wrap the archive + higherEnd = higher["offset"] + higher["size"] + seriesString = fh.read(higherEnd - higherFirstOffset) + fh.seek(higher["offset"]) + seriesString += fh.read(higherLastOffset - higher["offset"]) + + # Now we unpack the series data we just read + byteOrder, pointTypes = pointFormat[0], pointFormat[1:] + points = len(seriesString) // pointSize + seriesFormat = byteOrder + (pointTypes * points) + unpackedSeries = struct.unpack(seriesFormat, seriesString) + + # And finally we construct a list of values + neighborValues = [None] * points + currentInterval = lowerIntervalStart + step = higher["secondsPerPoint"] + + for i in xrange(0, len(unpackedSeries), 2): + pointTime = unpackedSeries[i] + if pointTime == currentInterval: + neighborValues[i // 2] = unpackedSeries[i + 1] + currentInterval += step + + # Propagate aggregateValue to propagate from neighborValues if we have enough known points + knownValues = [v for v in neighborValues if v is not None] + if not knownValues: + return False + + knownPercent = float(len(knownValues)) / float(len(neighborValues)) + if knownPercent >= xff: # we have enough data to propagate a value! + aggregateValue = aggregate(aggregationMethod, knownValues) + myPackedPoint = struct.pack( + pointFormat, lowerIntervalStart, aggregateValue + ) + fh.seek(lower["offset"]) + packedPoint = fh.read(pointSize) + (lowerBaseInterval, lowerBaseValue) = struct.unpack( + pointFormat, packedPoint + ) + + if ( + lowerBaseInterval == 0 + ): # First propagated update to this lower archive + fh.seek(lower["offset"]) + fh.write(myPackedPoint) + else: # Not our first propagated update to this lower archive + timeDistance = lowerIntervalStart - lowerBaseInterval + pointDistance = timeDistance // lower["secondsPerPoint"] + byteDistance = pointDistance * pointSize + lowerOffset = lower["offset"] + (byteDistance % lower["size"]) + fh.seek(lowerOffset) + fh.write(myPackedPoint) + + return True + + else: + return False -def update(path,value,timestamp=None): - """update(path,value,timestamp=None) + +def update(path, value, timestamp=None): + """update(path,value,timestamp=None) path is a string value is a float timestamp is either an int or float """ - value = float(value) - fh = None - try: - fh = open(path,'r+b') - return file_update(fh, value, timestamp) - finally: - if fh: - fh.close() + value = float(value) + fh = None + try: + fh = open(path, "r+b") + return file_update(fh, value, timestamp) + finally: + if fh: + fh.close() -def file_update(fh, value, timestamp): - if LOCK: - fcntl.flock( fh.fileno(), fcntl.LOCK_EX ) - - header = __readHeader(fh) - now = int( time.time() ) - if timestamp is None: - timestamp = now - - timestamp = int(timestamp) - diff = now - timestamp - if not ((diff < header['maxRetention']) and diff >= 0): - raise TimestampNotCovered("Timestamp not covered by any archives in " - "this database.") - - for i,archive in enumerate(header['archives']): #Find the highest-precision archive that covers timestamp - if archive['retention'] < diff: continue - lowerArchives = header['archives'][i+1:] #We'll pass on the update to these lower precision archives later - break - - #First we update the highest-precision archive - myInterval = timestamp - (timestamp % archive['secondsPerPoint']) - myPackedPoint = struct.pack(pointFormat,myInterval,value) - fh.seek(archive['offset']) - packedPoint = fh.read(pointSize) - (baseInterval,baseValue) = struct.unpack(pointFormat,packedPoint) - - if baseInterval == 0: #This file's first update - fh.seek(archive['offset']) - fh.write(myPackedPoint) - baseInterval,baseValue = myInterval,value - else: #Not our first update - timeDistance = myInterval - baseInterval - pointDistance = timeDistance // archive['secondsPerPoint'] - byteDistance = pointDistance * pointSize - myOffset = archive['offset'] + (byteDistance % archive['size']) - fh.seek(myOffset) - fh.write(myPackedPoint) - #Now we propagate the update to lower-precision archives - higher = archive - for lower in lowerArchives: - if not __propagate(fh, header, myInterval, higher, lower): - break - higher = lower +def file_update(fh, value, timestamp): + if LOCK: + fcntl.flock(fh.fileno(), fcntl.LOCK_EX) + + header = __readHeader(fh) + now = int(time.time()) + if timestamp is None: + timestamp = now + + timestamp = int(timestamp) + diff = now - timestamp + if not ((diff < header["maxRetention"]) and diff >= 0): + raise TimestampNotCovered( + "Timestamp not covered by any archives in " "this database." + ) + + for i, archive in enumerate( + header["archives"] + ): # Find the highest-precision archive that covers timestamp + if archive["retention"] < diff: + continue + lowerArchives = header["archives"][ + i + 1 : + ] # We'll pass on the update to these lower precision archives later + break - if AUTOFLUSH: - fh.flush() - os.fsync(fh.fileno()) + # First we update the highest-precision archive + myInterval = timestamp - (timestamp % archive["secondsPerPoint"]) + myPackedPoint = struct.pack(pointFormat, myInterval, value) + fh.seek(archive["offset"]) + packedPoint = fh.read(pointSize) + (baseInterval, baseValue) = struct.unpack(pointFormat, packedPoint) + + if baseInterval == 0: # This file's first update + fh.seek(archive["offset"]) + fh.write(myPackedPoint) + baseInterval, baseValue = myInterval, value + else: # Not our first update + timeDistance = myInterval - baseInterval + pointDistance = timeDistance // archive["secondsPerPoint"] + byteDistance = pointDistance * pointSize + myOffset = archive["offset"] + (byteDistance % archive["size"]) + fh.seek(myOffset) + fh.write(myPackedPoint) + + # Now we propagate the update to lower-precision archives + higher = archive + for lower in lowerArchives: + if not __propagate(fh, header, myInterval, higher, lower): + break + higher = lower + if AUTOFLUSH: + fh.flush() + os.fsync(fh.fileno()) -def update_many(path,points): - """update_many(path,points) +def update_many(path, points): + """update_many(path,points) path is a string points is a list of (timestamp,value) points """ - if not points: return - points = [ (int(t),float(v)) for (t,v) in points] - points.sort(key=lambda p: p[0],reverse=True) #order points by timestamp, newest first - fh = None - try: - fh = open(path,'r+b') - return file_update_many(fh, points) - finally: - if fh: - fh.close() + if not points: + return + points = [(int(t), float(v)) for (t, v) in points] + points.sort( + key=lambda p: p[0], reverse=True + ) # order points by timestamp, newest first + fh = None + try: + fh = open(path, "r+b") + return file_update_many(fh, points) + finally: + if fh: + fh.close() def file_update_many(fh, points): - if LOCK: - fcntl.flock( fh.fileno(), fcntl.LOCK_EX ) - - header = __readHeader(fh) - now = int( time.time() ) - archives = iter( header['archives'] ) - currentArchive = next(archives) - currentPoints = [] - - for point in points: - age = now - point[0] - - while currentArchive['retention'] < age: #we can't fit any more points in this archive - if currentPoints: #commit all the points we've found that it can fit - currentPoints.reverse() #put points in chronological order - __archive_update_many(fh,header,currentArchive,currentPoints) - currentPoints = [] - try: - currentArchive = next(archives) - except StopIteration: - currentArchive = None - break - - if not currentArchive: - break #drop remaining points that don't fit in the database - - currentPoints.append(point) - - if currentArchive and currentPoints: #don't forget to commit after we've checked all the archives - currentPoints.reverse() - __archive_update_many(fh,header,currentArchive,currentPoints) - - if AUTOFLUSH: - fh.flush() - os.fsync(fh.fileno()) - - - -def __archive_update_many(fh,header,archive,points): - step = archive['secondsPerPoint'] - alignedPoints = [ (timestamp - (timestamp % step), value) - for (timestamp,value) in points ] - alignedPoints = dict(alignedPoints).items() # Take the last val of duplicates - #Create a packed string for each contiguous sequence of points - packedStrings = [] - previousInterval = None - currentString = b"" - for (interval,value) in alignedPoints: - if (not previousInterval) or (interval == previousInterval + step): - currentString += struct.pack(pointFormat,interval,value) - previousInterval = interval - else: - numberOfPoints = len(currentString) // pointSize - startInterval = previousInterval - (step * (numberOfPoints-1)) - packedStrings.append( (startInterval,currentString) ) - currentString = struct.pack(pointFormat,interval,value) - previousInterval = interval - if currentString: - numberOfPoints = len(currentString) // pointSize - startInterval = previousInterval - (step * (numberOfPoints-1)) - packedStrings.append( (startInterval,currentString) ) - - #Read base point and determine where our writes will start - fh.seek(archive['offset']) - packedBasePoint = fh.read(pointSize) - (baseInterval,baseValue) = struct.unpack(pointFormat,packedBasePoint) - if baseInterval == 0: #This file's first update - baseInterval = packedStrings[0][0] #use our first string as the base, so we start at the start - - #Write all of our packed strings in locations determined by the baseInterval - for (interval,packedString) in packedStrings: - timeDistance = interval - baseInterval - pointDistance = timeDistance // step - byteDistance = pointDistance * pointSize - myOffset = archive['offset'] + (byteDistance % archive['size']) - fh.seek(myOffset) - archiveEnd = archive['offset'] + archive['size'] - bytesBeyond = (myOffset + len(packedString)) - archiveEnd - - if bytesBeyond > 0: - fh.write( packedString[:-bytesBeyond] ) - assert fh.tell() == archiveEnd, "archiveEnd=%d fh.tell=%d bytesBeyond=%d len(packedString)=%d" % (archiveEnd,fh.tell(),bytesBeyond,len(packedString)) - fh.seek( archive['offset'] ) - fh.write( packedString[-bytesBeyond:] ) #safe because it can't exceed the archive (retention checking logic above) - else: - fh.write(packedString) - - #Now we propagate the updates to lower-precision archives - higher = archive - lowerArchives = [arc for arc in header['archives'] if arc['secondsPerPoint'] > archive['secondsPerPoint']] - - for lower in lowerArchives: - fit = lambda i: i - (i % lower['secondsPerPoint']) - lowerIntervals = [fit(p[0]) for p in alignedPoints] - uniqueLowerIntervals = set(lowerIntervals) - propagateFurther = False - for interval in uniqueLowerIntervals: - if __propagate(fh, header, interval, higher, lower): - propagateFurther = True + if LOCK: + fcntl.flock(fh.fileno(), fcntl.LOCK_EX) + + header = __readHeader(fh) + now = int(time.time()) + archives = iter(header["archives"]) + currentArchive = next(archives) + currentPoints = [] + + for point in points: + age = now - point[0] + + while ( + currentArchive["retention"] < age + ): # we can't fit any more points in this archive + if ( + currentPoints + ): # commit all the points we've found that it can fit + currentPoints.reverse() # put points in chronological order + __archive_update_many( + fh, header, currentArchive, currentPoints + ) + currentPoints = [] + try: + currentArchive = next(archives) + except StopIteration: + currentArchive = None + break + + if not currentArchive: + break # drop remaining points that don't fit in the database + + currentPoints.append(point) + + if ( + currentArchive and currentPoints + ): # don't forget to commit after we've checked all the archives + currentPoints.reverse() + __archive_update_many(fh, header, currentArchive, currentPoints) - if not propagateFurther: - break - higher = lower + if AUTOFLUSH: + fh.flush() + os.fsync(fh.fileno()) + + +def __archive_update_many(fh, header, archive, points): + step = archive["secondsPerPoint"] + alignedPoints = [ + (timestamp - (timestamp % step), value) + for (timestamp, value) in points + ] + alignedPoints = dict( + alignedPoints + ).items() # Take the last val of duplicates + # Create a packed string for each contiguous sequence of points + packedStrings = [] + previousInterval = None + currentString = b"" + for (interval, value) in alignedPoints: + if (not previousInterval) or (interval == previousInterval + step): + currentString += struct.pack(pointFormat, interval, value) + previousInterval = interval + else: + numberOfPoints = len(currentString) // pointSize + startInterval = previousInterval - (step * (numberOfPoints - 1)) + packedStrings.append((startInterval, currentString)) + currentString = struct.pack(pointFormat, interval, value) + previousInterval = interval + if currentString: + numberOfPoints = len(currentString) // pointSize + startInterval = previousInterval - (step * (numberOfPoints - 1)) + packedStrings.append((startInterval, currentString)) + + # Read base point and determine where our writes will start + fh.seek(archive["offset"]) + packedBasePoint = fh.read(pointSize) + (baseInterval, baseValue) = struct.unpack(pointFormat, packedBasePoint) + if baseInterval == 0: # This file's first update + baseInterval = packedStrings[0][ + 0 + ] # use our first string as the base, so we start at the start + + # Write all of our packed strings in locations determined by the baseInterval + for (interval, packedString) in packedStrings: + timeDistance = interval - baseInterval + pointDistance = timeDistance // step + byteDistance = pointDistance * pointSize + myOffset = archive["offset"] + (byteDistance % archive["size"]) + fh.seek(myOffset) + archiveEnd = archive["offset"] + archive["size"] + bytesBeyond = (myOffset + len(packedString)) - archiveEnd + + if bytesBeyond > 0: + fh.write(packedString[:-bytesBeyond]) + assert fh.tell() == archiveEnd, ( + "archiveEnd=%d fh.tell=%d bytesBeyond=%d len(packedString)=%d" + % (archiveEnd, fh.tell(), bytesBeyond, len(packedString)) + ) + fh.seek(archive["offset"]) + fh.write( + packedString[-bytesBeyond:] + ) # safe because it can't exceed the archive (retention checking logic above) + else: + fh.write(packedString) + + # Now we propagate the updates to lower-precision archives + higher = archive + lowerArchives = [ + arc + for arc in header["archives"] + if arc["secondsPerPoint"] > archive["secondsPerPoint"] + ] + + for lower in lowerArchives: + fit = lambda i: i - (i % lower["secondsPerPoint"]) + lowerIntervals = [fit(p[0]) for p in alignedPoints] + uniqueLowerIntervals = set(lowerIntervals) + propagateFurther = False + for interval in uniqueLowerIntervals: + if __propagate(fh, header, interval, higher, lower): + propagateFurther = True + + if not propagateFurther: + break + higher = lower def info(path): - """info(path) + """info(path) path is a string """ - fh = None - try: - fh = open(path,'rb') - return __readHeader(fh) - finally: - if fh: - fh.close() - return None - -def fetch(path,fromTime,untilTime=None,now=None): - """fetch(path,fromTime,untilTime=None) + fh = None + try: + fh = open(path, "rb") + return __readHeader(fh) + finally: + if fh: + fh.close() + return None + + +def fetch(path, fromTime, untilTime=None, now=None): + """fetch(path,fromTime,untilTime=None) path is a string fromTime is an epoch time @@ -750,188 +868,225 @@ def fetch(path,fromTime,untilTime=None,now=None): Returns None if no data can be returned """ - fh = None - try: - fh = open(path,'rb') - return file_fetch(fh, fromTime, untilTime, now) - finally: - if fh: - fh.close() - -def file_fetch(fh, fromTime, untilTime, now = None): - header = __readHeader(fh) - if now is None: - now = int( time.time() ) - if untilTime is None: - untilTime = now - fromTime = int(fromTime) - untilTime = int(untilTime) - - # Here we try and be flexible and return as much data as we can. - # If the range of data is from too far in the past or fully in the future, we - # return nothing - if (fromTime > untilTime): - raise InvalidTimeInterval("Invalid time interval: from time '%s' is after until time '%s'" % (fromTime, untilTime)) - - oldestTime = now - header['maxRetention'] - # Range is in the future - if fromTime > now: - return None - # Range is beyond retention - if untilTime < oldestTime: - return None - # Range requested is partially beyond retention, adjust - if fromTime < oldestTime: - fromTime = oldestTime - # Range is partially in the future, adjust - if untilTime > now: - untilTime = now - - diff = now - fromTime - for archive in header['archives']: - if archive['retention'] >= diff: - break + fh = None + try: + fh = open(path, "rb") + return file_fetch(fh, fromTime, untilTime, now) + finally: + if fh: + fh.close() + + +def file_fetch(fh, fromTime, untilTime, now=None): + header = __readHeader(fh) + if now is None: + now = int(time.time()) + if untilTime is None: + untilTime = now + fromTime = int(fromTime) + untilTime = int(untilTime) + + # Here we try and be flexible and return as much data as we can. + # If the range of data is from too far in the past or fully in the future, we + # return nothing + if fromTime > untilTime: + raise InvalidTimeInterval( + "Invalid time interval: from time '%s' is after until time '%s'" + % (fromTime, untilTime) + ) + + oldestTime = now - header["maxRetention"] + # Range is in the future + if fromTime > now: + return None + # Range is beyond retention + if untilTime < oldestTime: + return None + # Range requested is partially beyond retention, adjust + if fromTime < oldestTime: + fromTime = oldestTime + # Range is partially in the future, adjust + if untilTime > now: + untilTime = now + + diff = now - fromTime + for archive in header["archives"]: + if archive["retention"] >= diff: + break + + return __archive_fetch(fh, archive, fromTime, untilTime) - return __archive_fetch(fh, archive, fromTime, untilTime) def __archive_fetch(fh, archive, fromTime, untilTime): - """ + """ Fetch data from a single archive. Note that checks for validity of the time period requested happen above this level so it's possible to wrap around the archive on a read and request data older than the archive's retention """ - fromInterval = int( fromTime - (fromTime % archive['secondsPerPoint']) ) + archive['secondsPerPoint'] - untilInterval = int( untilTime - (untilTime % archive['secondsPerPoint']) ) + archive['secondsPerPoint'] - if fromInterval == untilInterval: - # Zero-length time range: always include the next point - untilInterval += archive['secondsPerPoint'] - fh.seek(archive['offset']) - packedPoint = fh.read(pointSize) - (baseInterval,baseValue) = struct.unpack(pointFormat,packedPoint) - - if baseInterval == 0: - step = archive['secondsPerPoint'] - points = (untilInterval - fromInterval) // step - timeInfo = (fromInterval,untilInterval,step) - valueList = [None] * points - return (timeInfo,valueList) - - #Determine fromOffset - timeDistance = fromInterval - baseInterval - pointDistance = timeDistance // archive['secondsPerPoint'] - byteDistance = pointDistance * pointSize - fromOffset = archive['offset'] + (byteDistance % archive['size']) - - #Determine untilOffset - timeDistance = untilInterval - baseInterval - pointDistance = timeDistance // archive['secondsPerPoint'] - byteDistance = pointDistance * pointSize - untilOffset = archive['offset'] + (byteDistance % archive['size']) - - #Read all the points in the interval - fh.seek(fromOffset) - if fromOffset < untilOffset: #If we don't wrap around the archive - seriesString = fh.read(untilOffset - fromOffset) - else: #We do wrap around the archive, so we need two reads - archiveEnd = archive['offset'] + archive['size'] - seriesString = fh.read(archiveEnd - fromOffset) - fh.seek(archive['offset']) - seriesString += fh.read(untilOffset - archive['offset']) - - #Now we unpack the series data we just read (anything faster than unpack?) - byteOrder,pointTypes = pointFormat[0],pointFormat[1:] - points = len(seriesString) // pointSize - seriesFormat = byteOrder + (pointTypes * points) - unpackedSeries = struct.unpack(seriesFormat, seriesString) - - #And finally we construct a list of values (optimize this!) - valueList = [None] * points #pre-allocate entire list for speed - currentInterval = fromInterval - step = archive['secondsPerPoint'] - - for i in xrange(0,len(unpackedSeries),2): - pointTime = unpackedSeries[i] - if pointTime == currentInterval: - pointValue = unpackedSeries[i+1] - valueList[i//2] = pointValue #in-place reassignment is faster than append() - currentInterval += step - - timeInfo = (fromInterval,untilInterval,step) - return (timeInfo,valueList) + fromInterval = ( + int(fromTime - (fromTime % archive["secondsPerPoint"])) + + archive["secondsPerPoint"] + ) + untilInterval = ( + int(untilTime - (untilTime % archive["secondsPerPoint"])) + + archive["secondsPerPoint"] + ) + if fromInterval == untilInterval: + # Zero-length time range: always include the next point + untilInterval += archive["secondsPerPoint"] + fh.seek(archive["offset"]) + packedPoint = fh.read(pointSize) + (baseInterval, baseValue) = struct.unpack(pointFormat, packedPoint) + + if baseInterval == 0: + step = archive["secondsPerPoint"] + points = (untilInterval - fromInterval) // step + timeInfo = (fromInterval, untilInterval, step) + valueList = [None] * points + return (timeInfo, valueList) + + # Determine fromOffset + timeDistance = fromInterval - baseInterval + pointDistance = timeDistance // archive["secondsPerPoint"] + byteDistance = pointDistance * pointSize + fromOffset = archive["offset"] + (byteDistance % archive["size"]) + + # Determine untilOffset + timeDistance = untilInterval - baseInterval + pointDistance = timeDistance // archive["secondsPerPoint"] + byteDistance = pointDistance * pointSize + untilOffset = archive["offset"] + (byteDistance % archive["size"]) + + # Read all the points in the interval + fh.seek(fromOffset) + if fromOffset < untilOffset: # If we don't wrap around the archive + seriesString = fh.read(untilOffset - fromOffset) + else: # We do wrap around the archive, so we need two reads + archiveEnd = archive["offset"] + archive["size"] + seriesString = fh.read(archiveEnd - fromOffset) + fh.seek(archive["offset"]) + seriesString += fh.read(untilOffset - archive["offset"]) + + # Now we unpack the series data we just read (anything faster than unpack?) + byteOrder, pointTypes = pointFormat[0], pointFormat[1:] + points = len(seriesString) // pointSize + seriesFormat = byteOrder + (pointTypes * points) + unpackedSeries = struct.unpack(seriesFormat, seriesString) + + # And finally we construct a list of values (optimize this!) + valueList = [None] * points # pre-allocate entire list for speed + currentInterval = fromInterval + step = archive["secondsPerPoint"] + + for i in xrange(0, len(unpackedSeries), 2): + pointTime = unpackedSeries[i] + if pointTime == currentInterval: + pointValue = unpackedSeries[i + 1] + valueList[ + i // 2 + ] = pointValue # in-place reassignment is faster than append() + currentInterval += step + + timeInfo = (fromInterval, untilInterval, step) + return (timeInfo, valueList) + def merge(path_from, path_to): - """ Merges the data from one whisper file into another. Each file must have + """ Merges the data from one whisper file into another. Each file must have the same archive configuration """ - fh_from = open(path_from, 'rb') - fh_to = open(path_to, 'rb+') - return file_merge(fh_from, fh_to) + fh_from = open(path_from, "rb") + fh_to = open(path_to, "rb+") + return file_merge(fh_from, fh_to) + def file_merge(fh_from, fh_to): - headerFrom = __readHeader(fh_from) - headerTo = __readHeader(fh_to) - - if headerFrom['archives'] != headerTo['archives']: - raise NotImplementedError("%s and %s archive configurations are unalike. " \ - "Resize the input before merging" % (fh_from.name, fh_to.name)) - - archives = headerFrom['archives'] - archives.sort(key=operator.itemgetter('retention')) - - now = int(time.time()) - untilTime = now - for archive in archives: - fromTime = now - archive['retention'] - (timeInfo, values) = __archive_fetch(fh_from, archive, fromTime, untilTime) - (start, end, archive_step) = timeInfo - pointsToWrite = list(ifilter( - lambda points: points[1] is not None, - izip(xrange(start, end, archive_step), values))) - __archive_update_many(fh_to, headerTo, archive, pointsToWrite) - untilTime = fromTime - fh_from.close() - fh_to.close() - -def diff(path_from, path_to, ignore_empty = False): - """ Compare two whisper databases. Each file must have the same archive configuration """ - fh_from = open(path_from, 'rb') - fh_to = open(path_to, 'rb') - diffs = file_diff(fh_from, fh_to, ignore_empty) - fh_to.close() - fh_from.close() - return diffs - -def file_diff(fh_from, fh_to, ignore_empty = False): - headerFrom = __readHeader(fh_from) - headerTo = __readHeader(fh_to) - - if headerFrom['archives'] != headerTo['archives']: - # TODO: Add specific whisper-resize commands to right size things - raise NotImplementedError("%s and %s archive configurations are unalike. " \ - "Resize the input before diffing" % (fh_from.name, fh_to.name)) - - archives = headerFrom['archives'] - archives.sort(key=operator.itemgetter('retention')) - - archive_diffs = [] - - now = int(time.time()) - untilTime = now - for archive_number, archive in enumerate(archives): - diffs = [] - startTime = now - archive['retention'] - (fromTimeInfo, fromValues) = __archive_fetch(fh_from, archive, startTime, untilTime) - (toTimeInfo, toValues) = __archive_fetch(fh_to, archive, startTime, untilTime) - (start, end, archive_step) = ( min(fromTimeInfo[0],toTimeInfo[0]), max(fromTimeInfo[1],toTimeInfo[1]), min(fromTimeInfo[2],toTimeInfo[2]) ) - - points = map(lambda s: (s * archive_step + start,fromValues[s],toValues[s]), xrange(0,(end - start) // archive_step)) - if ignore_empty: - points = [p for p in points if p[1] != None and p[2] != None] - else: - points = [p for p in points if p[1] != None or p[2] != None] + headerFrom = __readHeader(fh_from) + headerTo = __readHeader(fh_to) - diffs = [p for p in points if p[1] != p[2]] + if headerFrom["archives"] != headerTo["archives"]: + raise NotImplementedError( + "%s and %s archive configurations are unalike. " + "Resize the input before merging" % (fh_from.name, fh_to.name) + ) - archive_diffs.append( (archive_number, diffs, points.__len__()) ) - untilTime = startTime - return archive_diffs + archives = headerFrom["archives"] + archives.sort(key=operator.itemgetter("retention")) + + now = int(time.time()) + untilTime = now + for archive in archives: + fromTime = now - archive["retention"] + (timeInfo, values) = __archive_fetch( + fh_from, archive, fromTime, untilTime + ) + (start, end, archive_step) = timeInfo + pointsToWrite = list( + ifilter( + lambda points: points[1] is not None, + izip(xrange(start, end, archive_step), values), + ) + ) + __archive_update_many(fh_to, headerTo, archive, pointsToWrite) + untilTime = fromTime + fh_from.close() + fh_to.close() + + +def diff(path_from, path_to, ignore_empty=False): + """ Compare two whisper databases. Each file must have the same archive configuration """ + fh_from = open(path_from, "rb") + fh_to = open(path_to, "rb") + diffs = file_diff(fh_from, fh_to, ignore_empty) + fh_to.close() + fh_from.close() + return diffs + + +def file_diff(fh_from, fh_to, ignore_empty=False): + headerFrom = __readHeader(fh_from) + headerTo = __readHeader(fh_to) + + if headerFrom["archives"] != headerTo["archives"]: + # TODO: Add specific whisper-resize commands to right size things + raise NotImplementedError( + "%s and %s archive configurations are unalike. " + "Resize the input before diffing" % (fh_from.name, fh_to.name) + ) + + archives = headerFrom["archives"] + archives.sort(key=operator.itemgetter("retention")) + + archive_diffs = [] + + now = int(time.time()) + untilTime = now + for archive_number, archive in enumerate(archives): + diffs = [] + startTime = now - archive["retention"] + (fromTimeInfo, fromValues) = __archive_fetch( + fh_from, archive, startTime, untilTime + ) + (toTimeInfo, toValues) = __archive_fetch( + fh_to, archive, startTime, untilTime + ) + (start, end, archive_step) = ( + min(fromTimeInfo[0], toTimeInfo[0]), + max(fromTimeInfo[1], toTimeInfo[1]), + min(fromTimeInfo[2], toTimeInfo[2]), + ) + + points = map( + lambda s: (s * archive_step + start, fromValues[s], toValues[s]), + xrange(0, (end - start) // archive_step), + ) + if ignore_empty: + points = [p for p in points if p[1] != None and p[2] != None] + else: + points = [p for p in points if p[1] != None or p[2] != None] + + diffs = [p for p in points if p[1] != p[2]] + + archive_diffs.append((archive_number, diffs, points.__len__())) + untilTime = startTime + return archive_diffs diff --git a/graphite_api/app.py b/graphite_api/app.py index 6a46b13..05d9218 100644 --- a/graphite_api/app.py +++ b/graphite_api/app.py @@ -26,25 +26,25 @@ def jsonify(data, status=200, headers=None): if headers is None: headers = {} - jsonp = RequestParams.get('jsonp', False) + jsonp = RequestParams.get("jsonp", False) body = json.dumps(data, cls=JSONEncoder) if jsonp: - headers['Content-Type'] = 'text/javascript' - body = '{0}({1})'.format(jsonp, body) + headers["Content-Type"] = "text/javascript" + body = "{0}({1})".format(jsonp, body) else: - headers['Content-Type'] = 'application/json' + headers["Content-Type"] = "application/json" return body, status, headers class Graphite(Flask): @property def store(self): - return self.config['GRAPHITE']['store'] + return self.config["GRAPHITE"]["store"] @property def functions(self): - return self.config['GRAPHITE']['functions'] + return self.config["GRAPHITE"]["functions"] @property def logger(self): @@ -58,32 +58,34 @@ def logger(self): configure(app) except Exception: import traceback + print(traceback.format_exc()) raise -methods = ('GET', 'POST') +methods = ("GET", "POST") # No-op routes, non-essential for creating dashboards -@app.route('/dashboard/find', methods=methods) +@app.route("/dashboard/find", methods=methods) def dashboard_find(): - return jsonify({'dashboards': []}) + return jsonify({"dashboards": []}) -@app.route('/dashboard/load/', methods=methods) +@app.route("/dashboard/load/", methods=methods) def dashboard_load(name): - return jsonify({'error': "Dashboard '{0}' does not exist.".format(name)}, - status=404) + return jsonify( + {"error": "Dashboard '{0}' does not exist.".format(name)}, status=404 + ) -@app.route('/events/get_data', methods=methods) +@app.route("/events/get_data", methods=methods) def events(): - return json.dumps([]), 200, {'Content-Type': 'application/json'} + return json.dumps([]), 200, {"Content-Type": "application/json"} # API calls that actually do something -@app.route('/metrics', methods=methods) -@app.route('/metrics/find', methods=methods) +@app.route("/metrics", methods=methods) +@app.route("/metrics/find", methods=methods) def metrics_find(): errors = {} from_time = None @@ -91,100 +93,96 @@ def metrics_find(): wildcards = False try: - wildcards = bool(int(RequestParams.get('wildcards', 0))) + wildcards = bool(int(RequestParams.get("wildcards", 0))) except ValueError: - errors['wildcards'] = 'must be 0 or 1.' + errors["wildcards"] = "must be 0 or 1." try: - from_time = int(RequestParams.get('from', -1)) + from_time = int(RequestParams.get("from", -1)) except ValueError: - errors['from'] = 'must be an epoch timestamp.' + errors["from"] = "must be an epoch timestamp." try: - until_time = int(RequestParams.get('until', -1)) + until_time = int(RequestParams.get("until", -1)) except ValueError: - errors['until'] = 'must be an epoch timestamp.' + errors["until"] = "must be an epoch timestamp." try: - node_position = int(RequestParams.get('position', -1)) + node_position = int(RequestParams.get("position", -1)) except ValueError: - errors['position'] = 'must be an integer,' + errors["position"] = "must be an integer," if from_time == -1: from_time = None if until_time == -1: until_time = None - format = RequestParams.get('format', 'treejson') - if format not in ['treejson', 'completer', 'nodelist', 'json']: - errors['format'] = 'unrecognized format: "{0}".'.format(format) + format = RequestParams.get("format", "treejson") + if format not in ["treejson", "completer", "nodelist", "json"]: + errors["format"] = 'unrecognized format: "{0}".'.format(format) - if 'query' not in RequestParams: - errors['query'] = 'this parameter is required.' + if "query" not in RequestParams: + errors["query"] = "this parameter is required." if errors: - return jsonify({'errors': errors}, status=400) + return jsonify({"errors": errors}, status=400) - query = RequestParams['query'] + query = RequestParams["query"] matches = sorted( app.store.find(query, from_time, until_time), - key=lambda node: node.name + key=lambda node: node.name, ) - base_path = query.rsplit('.', 1)[0] + '.' if '.' in query else '' + base_path = query.rsplit(".", 1)[0] + "." if "." in query else "" - if format == 'treejson': + if format == "treejson": data = tree_json(matches, base_path, wildcards=wildcards) - return ( - json.dumps(data), - 200, - {'Content-Type': 'application/json'} - ) - elif format == 'nodelist': + return (json.dumps(data), 200, {"Content-Type": "application/json"}) + elif format == "nodelist": found = set() for metric in matches: - nodes = metric.path.split('.') + nodes = metric.path.split(".") found.add(nodes[node_position]) - return jsonify({'nodes': sorted(found)}) - elif format == 'json': + return jsonify({"nodes": sorted(found)}) + elif format == "json": content = json_nodes(matches) return jsonify(content) results = [] for node in matches: node_info = { - 'path': node.path, - 'name': node.name, - 'is_leaf': int(node.is_leaf), # XXX Y was this cast to str + "path": node.path, + "name": node.name, + "is_leaf": int(node.is_leaf), # XXX Y was this cast to str } if not node.is_leaf: - node_info['path'] += '.' + node_info["path"] += "." results.append(node_info) if len(results) > 1 and wildcards: - results.append({'name': '*'}) + results.append({"name": "*"}) - return jsonify({'metrics': results}) + return jsonify({"metrics": results}) -@app.route('/metrics/expand', methods=methods) +@app.route("/metrics/expand", methods=methods) def metrics_expand(): errors = {} try: - group_by_expr = bool(int(RequestParams.get('groupByExpr', 0))) + group_by_expr = bool(int(RequestParams.get("groupByExpr", 0))) except ValueError: - errors['groupByExpr'] = 'must be 0 or 1.' + errors["groupByExpr"] = "must be 0 or 1." try: - leaves_only = bool(int(RequestParams.get('leavesOnly', 0))) + leaves_only = bool(int(RequestParams.get("leavesOnly", 0))) except ValueError: - errors['leavesOnly'] = 'must be 0 or 1.' + errors["leavesOnly"] = "must be 0 or 1." - if 'query' not in RequestParams: - errors['query'] = 'this parameter is required.' + if "query" not in RequestParams: + errors["query"] = "this parameter is required." if errors: - return jsonify({'errors': errors}, status=400) + return jsonify({"errors": errors}, status=400) results = defaultdict(set) - for query in RequestParams.getlist('query'): + for query in RequestParams.getlist("query"): for node in app.store.find(query): if node.is_leaf or not leaves_only: results[query].add(node.path) @@ -198,7 +196,7 @@ def metrics_expand(): new_results = new_results.union(value) results = sorted(new_results) - return jsonify({'results': results}) + return jsonify({"results": results}) def recurse(query, index): @@ -209,13 +207,13 @@ def recurse(query, index): if node.is_leaf: index.add(node.path) else: - recurse('{0}.*'.format(node.path), index) + recurse("{0}.*".format(node.path), index) -@app.route('/metrics/index.json', methods=methods) +@app.route("/metrics/index.json", methods=methods) def metrics_index(): index = set() - recurse('*', index) + recurse("*", index) return jsonify(sorted(index)) @@ -228,13 +226,13 @@ def prune_datapoints(series, max_datapoints, start, end): ) seconds_per_point = values_per_point * series.step nudge = ( - seconds_per_point + - (series.start % series.step) - - (series.start % seconds_per_point) + seconds_per_point + + (series.start % series.step) + - (series.start % seconds_per_point) ) series.start += nudge values_to_lose = nudge // series.step - del series[:values_to_lose-1] + del series[: values_to_lose - 1] series.consolidate(values_per_point) step = seconds_per_point else: @@ -242,55 +240,55 @@ def prune_datapoints(series, max_datapoints, start, end): timestamps = range(series.start, series.end + series.step, step) datapoints = zip(series, timestamps) - return {'target': series.name, 'datapoints': datapoints} + return {"target": series.name, "datapoints": datapoints} -@app.route('/render', methods=methods) +@app.route("/render", methods=methods) def render(): start = time.time() # Start with some defaults errors = {} - graph_options = { - 'width': 600, - 'height': 300, - } + graph_options = {"width": 600, "height": 300} request_options = {} # Fill in the request_options - graph_type = RequestParams.get('graphType', 'line') + graph_type = RequestParams.get("graphType", "line") # Fill in the request_options try: graph_class = GraphTypes[graph_type] - request_options['graphType'] = graph_type - request_options['graphClass'] = graph_class + request_options["graphType"] = graph_type + request_options["graphClass"] = graph_class except KeyError: - errors['graphType'] = ( - "Invalid graphType '{0}', must be one of '{1}'.".format( - graph_type, "', '".join(sorted(GraphTypes)))) - request_options['pieMode'] = RequestParams.get('pieMode', 'average') - targets = RequestParams.getlist('target') + errors[ + "graphType" + ] = "Invalid graphType '{0}', must be one of '{1}'.".format( + graph_type, "', '".join(sorted(GraphTypes)) + ) + request_options["pieMode"] = RequestParams.get("pieMode", "average") + targets = RequestParams.getlist("target") if not len(targets): - errors['target'] = 'This parameter is required.' - request_options['targets'] = targets - - if 'rawData' in RequestParams: - request_options['format'] = 'raw' - if 'format' in RequestParams: - request_options['format'] = RequestParams['format'] - if 'jsonp' in RequestParams: - request_options['jsonp'] = RequestParams['jsonp'] - if 'maxDataPoints' in RequestParams: + errors["target"] = "This parameter is required." + request_options["targets"] = targets + + if "rawData" in RequestParams: + request_options["format"] = "raw" + if "format" in RequestParams: + request_options["format"] = RequestParams["format"] + if "jsonp" in RequestParams: + request_options["jsonp"] = RequestParams["jsonp"] + if "maxDataPoints" in RequestParams: try: - request_options['maxDataPoints'] = int( - float(RequestParams['maxDataPoints'])) + request_options["maxDataPoints"] = int( + float(RequestParams["maxDataPoints"]) + ) except ValueError: - errors['maxDataPoints'] = 'Must be an integer.' - if 'noNullPoints' in RequestParams: - request_options['noNullPoints'] = True + errors["maxDataPoints"] = "Must be an integer." + if "noNullPoints" in RequestParams: + request_options["noNullPoints"] = True if errors: - return jsonify({'errors': errors}, status=400) + return jsonify({"errors": errors}, status=400) # Fill in the graph_options for opt in graph_class.customizable: @@ -304,48 +302,48 @@ def render(): try: value = float(value) except ValueError: - if value.lower() in ('true', 'false'): - value = value.lower() == 'true' - elif value.lower() == 'default' or not value: + if value.lower() in ("true", "false"): + value = value.lower() == "true" + elif value.lower() == "default" or not value: continue graph_options[opt] = value - tzinfo = pytz.timezone(app.config['TIME_ZONE']) - tz = RequestParams.get('tz') + tzinfo = pytz.timezone(app.config["TIME_ZONE"]) + tz = RequestParams.get("tz") if tz: try: tzinfo = pytz.timezone(tz) except pytz.UnknownTimeZoneError: - errors['tz'] = "Unknown timezone: '{0}'.".format(tz) - request_options['tzinfo'] = tzinfo + errors["tz"] = "Unknown timezone: '{0}'.".format(tz) + request_options["tzinfo"] = tzinfo # Get the time interval for time-oriented graph types - now = parseATTime(RequestParams.get('now', 'now'), tzinfo) - until_time = parseATTime(RequestParams.get('until', 'now'), tzinfo, now) - from_time = parseATTime(RequestParams.get('from', '-1d'), tzinfo, now) + now = parseATTime(RequestParams.get("now", "now"), tzinfo) + until_time = parseATTime(RequestParams.get("until", "now"), tzinfo, now) + from_time = parseATTime(RequestParams.get("from", "-1d"), tzinfo, now) start_time = min(from_time, until_time) end_time = max(from_time, until_time) if start_time == end_time: - errors['from'] = errors['until'] = 'Invalid empty time range' + errors["from"] = errors["until"] = "Invalid empty time range" - request_options['startTime'] = start_time - request_options['endTime'] = end_time - request_options['now'] = now + request_options["startTime"] = start_time + request_options["endTime"] = end_time + request_options["now"] = now template = dict() for key in RequestParams.keys(): - if key.startswith('template['): + if key.startswith("template["): template[key[9:-1]] = RequestParams.get(key) - request_options['template'] = template + request_options["template"] = template - use_cache = app.cache is not None and 'noCache' not in RequestParams - cache_timeout = RequestParams.get('cacheTimeout') + use_cache = app.cache is not None and "noCache" not in RequestParams + cache_timeout = RequestParams.get("cacheTimeout") if cache_timeout is not None: cache_timeout = int(cache_timeout) if errors: - return jsonify({'errors': errors}, status=400) + return jsonify({"errors": errors}, status=400) # Done with options. @@ -353,218 +351,249 @@ def render(): request_key = hash_request() response = app.cache.get(request_key) if response is not None: - logger.debug("cached response", time=(time.time() - start), - targets=targets) + logger.debug( + "cached response", time=(time.time() - start), targets=targets + ) return response - headers = { - 'Last-Modified': http_date(time.time()), - 'Expires': http_date(time.time() + (cache_timeout or 60)), - 'Cache-Control': 'max-age={0}'.format(cache_timeout or 60) - } if use_cache else { - 'Pragma': 'no-cache', - 'Cache-Control': 'no-cache', - } + headers = ( + { + "Last-Modified": http_date(time.time()), + "Expires": http_date(time.time() + (cache_timeout or 60)), + "Cache-Control": "max-age={0}".format(cache_timeout or 60), + } + if use_cache + else {"Pragma": "no-cache", "Cache-Control": "no-cache"} + ) context = { - 'startTime': request_options['startTime'], - 'endTime': request_options['endTime'], - 'now': request_options['now'], - 'tzinfo': request_options['tzinfo'], - 'template': request_options['template'], - 'data': [], + "startTime": request_options["startTime"], + "endTime": request_options["endTime"], + "now": request_options["now"], + "tzinfo": request_options["tzinfo"], + "template": request_options["template"], + "data": [], } # Gather all data to take advantage of backends with fetch_multi fdstart = time.time() paths = [] - for target in request_options['targets']: - if request_options['graphType'] == 'pie': - if ':' in target: + for target in request_options["targets"]: + if request_options["graphType"] == "pie": + if ":" in target: continue if target.strip(): paths += pathsFromTarget(context, target) data_store = fetchData(context, paths) logger.debug("fetched data", time=(time.time() - fdstart), paths=paths) - if request_options['graphType'] == 'pie': - for target in request_options['targets']: - if ':' in target: - name, value = target.split(':', 1) + if request_options["graphType"] == "pie": + for target in request_options["targets"]: + if ":" in target: + name, value = target.split(":", 1) try: value = float(value) except ValueError: - errors['target'] = "Invalid target: '{0}'.".format(target) - context['data'].append((name, value)) + errors["target"] = "Invalid target: '{0}'.".format(target) + context["data"].append((name, value)) else: series_list = evaluateTarget(context, target, data_store) for series in series_list: - func = app.functions[request_options['pieMode']] - context['data'].append((series.name, - func(context, series) or 0)) + func = app.functions[request_options["pieMode"]] + context["data"].append( + (series.name, func(context, series) or 0) + ) if errors: - return jsonify({'errors': errors}, status=400) + return jsonify({"errors": errors}, status=400) else: # graphType == 'line' - for target in request_options['targets']: + for target in request_options["targets"]: if not target.strip(): continue emstart = time.time() series_list = evaluateTarget(context, target, data_store) - logger.debug("evaluated metric", time=(time.time() - emstart), - target=target) - context['data'].extend(series_list) + logger.debug( + "evaluated metric", + time=(time.time() - emstart), + target=target, + ) + context["data"].extend(series_list) - request_options['format'] = request_options.get('format') + request_options["format"] = request_options.get("format") - if request_options['format'] == 'csv': + if request_options["format"] == "csv": response = BytesIO() if six.PY2 else StringIO() - writer = csv.writer(response, dialect='excel') - for series in context['data']: + writer = csv.writer(response, dialect="excel") + for series in context["data"]: for index, value in enumerate(series): ts = datetime.fromtimestamp( series.start + index * series.step, - request_options['tzinfo'] + request_options["tzinfo"], + ) + writer.writerow( + (series.name, ts.strftime("%Y-%m-%d %H:%M:%S"), value) ) - writer.writerow((series.name, - ts.strftime("%Y-%m-%d %H:%M:%S"), value)) response.seek(0) - headers['Content-Type'] = 'text/csv' + headers["Content-Type"] = "text/csv" response = (response.read(), 200, headers) if use_cache: app.cache.add(request_key, response, cache_timeout) return response - if request_options['format'] == 'json': + if request_options["format"] == "json": series_data = [] - if 'maxDataPoints' in request_options and any(context['data']): - start_time = min([s.start for s in context['data']]) - end_time = max([s.end for s in context['data']]) - for series in context['data']: - series_data.append(prune_datapoints( - series, request_options['maxDataPoints'], - start_time, end_time)) - elif 'noNullPoints' in request_options and any(context['data']): - for series in context['data']: + if "maxDataPoints" in request_options and any(context["data"]): + start_time = min([s.start for s in context["data"]]) + end_time = max([s.end for s in context["data"]]) + for series in context["data"]: + series_data.append( + prune_datapoints( + series, + request_options["maxDataPoints"], + start_time, + end_time, + ) + ) + elif "noNullPoints" in request_options and any(context["data"]): + for series in context["data"]: values = [] for (index, v) in enumerate(series): if v is not None: timestamp = series.start + (index * series.step) values.append((v, timestamp)) if len(values) > 0: - series_data.append({'target': series.name, - 'datapoints': values}) + series_data.append( + {"target": series.name, "datapoints": values} + ) else: - for series in context['data']: - timestamps = range(series.start, series.end + series.step, - series.step) + for series in context["data"]: + timestamps = range( + series.start, series.end + series.step, series.step + ) datapoints = zip(series, timestamps) - series_data.append({'target': series.name, - 'datapoints': datapoints}) + series_data.append( + {"target": series.name, "datapoints": datapoints} + ) response = jsonify(series_data, headers=headers) if use_cache: app.cache.add(request_key, response, cache_timeout) - logger.debug("rendered json", time=(time.time() - start), - targets=targets) + logger.debug( + "rendered json", time=(time.time() - start), targets=targets + ) return response - if request_options['format'] == 'dygraph': + if request_options["format"] == "dygraph": series_data = {} - labels = ['Time'] - if any(context['data']): - datapoints = [[ts * 1000] - for ts in range(context['data'][0].start, - context['data'][0].end, - context['data'][0].step)] - for series in context['data']: + labels = ["Time"] + if any(context["data"]): + datapoints = [ + [ts * 1000] + for ts in range( + context["data"][0].start, + context["data"][0].end, + context["data"][0].step, + ) + ] + for series in context["data"]: labels.append(series.name) for i, point in enumerate(series): datapoints[i].append(point) - series_data = {'labels': labels, 'data': datapoints} + series_data = {"labels": labels, "data": datapoints} - logger.debug("rendered dygraph", time=(time.time() - start), - targets=targets) + logger.debug( + "rendered dygraph", + time=(time.time() - start), + targets=targets, + ) return jsonify(series_data, headers=headers) - if request_options['format'] == 'rickshaw': + if request_options["format"] == "rickshaw": series_data = [] - for series in context['data']: + for series in context["data"]: timestamps = range(series.start, series.end, series.step) - datapoints = [{'x': x, 'y': y} - for x, y in zip(timestamps, series)] - series_data.append(dict(target=series.name, - datapoints=datapoints)) - logger.debug("rendered rickshaw", time=(time.time() - start), - targets=targets) + datapoints = [ + {"x": x, "y": y} for x, y in zip(timestamps, series) + ] + series_data.append( + dict(target=series.name, datapoints=datapoints) + ) + logger.debug( + "rendered rickshaw", + time=(time.time() - start), + targets=targets, + ) return jsonify(series_data, headers=headers) - if request_options['format'] == 'raw': + if request_options["format"] == "raw": response = StringIO() - for series in context['data']: - response.write(u"%s,%d,%d,%d|" % ( - series.name, series.start, series.end, series.step)) - response.write(u','.join(map(repr, series))) - response.write(u'\n') + for series in context["data"]: + response.write( + u"%s,%d,%d,%d|" + % (series.name, series.start, series.end, series.step) + ) + response.write(u",".join(map(repr, series))) + response.write(u"\n") response.seek(0) - headers['Content-Type'] = 'text/plain' + headers["Content-Type"] = "text/plain" response = (response.read(), 200, headers) if use_cache: app.cache.add(request_key, response, cache_timeout) - logger.debug("rendered rawData", time=(time.time() - start), - targets=targets) + logger.debug( + "rendered rawData", + time=(time.time() - start), + targets=targets, + ) return response - if request_options['format'] == 'svg': - graph_options['outputFormat'] = 'svg' - elif request_options['format'] == 'pdf': - graph_options['outputFormat'] = 'pdf' + if request_options["format"] == "svg": + graph_options["outputFormat"] = "svg" + elif request_options["format"] == "pdf": + graph_options["outputFormat"] = "pdf" - graph_options['data'] = context['data'] - image = doImageRender(request_options['graphClass'], graph_options) + graph_options["data"] = context["data"] + image = doImageRender(request_options["graphClass"], graph_options) - use_svg = graph_options.get('outputFormat') == 'svg' + use_svg = graph_options.get("outputFormat") == "svg" - if use_svg and 'jsonp' in request_options: - headers['Content-Type'] = 'text/javascript' - response = ('{0}({1})'.format(request_options['jsonp'], - json.dumps(image.decode('utf-8'))), - 200, headers) + if use_svg and "jsonp" in request_options: + headers["Content-Type"] = "text/javascript" + response = ( + "{0}({1})".format( + request_options["jsonp"], json.dumps(image.decode("utf-8")) + ), + 200, + headers, + ) else: if use_svg: - ctype = 'image/svg+xml' - elif graph_options.get('outputFormat') == 'pdf': - ctype = 'application/x-pdf' + ctype = "image/svg+xml" + elif graph_options.get("outputFormat") == "pdf": + ctype = "application/x-pdf" else: - ctype = 'image/png' - headers['Content-Type'] = ctype + ctype = "image/png" + headers["Content-Type"] = ctype response = image, 200, headers if use_cache: app.cache.add(request_key, response, cache_timeout) - logger.debug("rendered graph", time=(time.time() - start), targets=targets) + logger.debug( + "rendered graph", time=(time.time() - start), targets=targets + ) return response def tree_json(nodes, base_path, wildcards=False): results = [] - branchNode = { - 'allowChildren': 1, - 'expandable': 1, - 'leaf': 0, - } - leafNode = { - 'allowChildren': 0, - 'expandable': 0, - 'leaf': 1, - } + branchNode = {"allowChildren": 1, "expandable": 1, "leaf": 0} + leafNode = {"allowChildren": 0, "expandable": 0, "leaf": 1} # Add a wildcard node if appropriate if len(nodes) > 1 and wildcards: - wildcardNode = {'text': '*', 'id': base_path + '*'} + wildcardNode = {"text": "*", "id": base_path + "*"} if any(not n.is_leaf for n in nodes): wildcardNode.update(branchNode) @@ -583,8 +612,8 @@ def tree_json(nodes, base_path, wildcards=False): found.add(node.name) resultNode = { - 'text': str(node.name), - 'id': base_path + str(node.name), + "text": str(node.name), + "id": base_path + str(node.name), } if node.is_leaf: @@ -603,19 +632,15 @@ def json_nodes(nodes): nodes_info = [] for node in nodes: - info = { - 'path': node.path, - 'is_leaf': node.is_leaf, - 'intervals': [], - } + info = {"path": node.path, "is_leaf": node.is_leaf, "intervals": []} if node.is_leaf: for i in node.intervals: - interval = {'start': i.start, 'end': i.end} - info['intervals'].append(interval) + interval = {"start": i.start, "end": i.end} + info["intervals"].append(interval) nodes_info.append(info) - return sorted(nodes_info, key=lambda item: item['path']) + return sorted(nodes_info, key=lambda item: item["path"]) def doImageRender(graphClass, graphOptions): diff --git a/graphite_api/carbonlink.py b/graphite_api/carbonlink.py index e040a0e..3b07e0a 100644 --- a/graphite_api/carbonlink.py +++ b/graphite_api/carbonlink.py @@ -3,7 +3,6 @@ import socket import struct import time - from hashlib import md5 from importlib import import_module from io import BytesIO @@ -15,24 +14,25 @@ logger = get_logger() pickle_safe = { - 'copy_reg': set(['_reconstructor']), - '__builtin__': set(['object', 'list']), - 'collections': set(['deque']), -} -renames = { - 'copy_reg': 'copyreg', - '__builtin__': 'builtins', + "copy_reg": set(["_reconstructor"]), + "__builtin__": set(["object", "list"]), + "collections": set(["deque"]), } +renames = {"copy_reg": "copyreg", "__builtin__": "builtins"} try: import pyhash + hasher = pyhash.fnv1a_32() - def fnv32a(string, seed=0x811c9dc5): + def fnv32a(string, seed=0x811C9DC5): return hasher(string, seed=seed) + + except ImportError: - def fnv32a(string, seed=0x811c9dc5): + + def fnv32a(string, seed=0x811C9DC5): """ FNV-1a Hash (http://isthe.com/chongo/tech/comp/fnv/) in Python. Taken from https://gist.github.com/vaiorabbit/5670985 @@ -49,17 +49,20 @@ def fnv32a(string, seed=0x811c9dc5): def allowed_module(module, name): if module not in pickle_safe: raise pickle.UnpicklingError( - 'Attempting to unpickle unsafe module %s' % module) + "Attempting to unpickle unsafe module %s" % module + ) if name not in pickle_safe[module]: raise pickle.UnpicklingError( - 'Attempting to unpickle unsafe class %s' % name) + "Attempting to unpickle unsafe class %s" % name + ) if module in renames: - module = 'six.moves.{0}'.format(renames[module]) + module = "six.moves.{0}".format(renames[module]) mod = import_module(module) return getattr(mod, name) if six.PY2: + class SafeUnpickler(object): @classmethod def find_class(cls, module, name): @@ -70,7 +73,10 @@ def loads(cls, s): obj = pickle.Unpickler(BytesIO(s)) obj.find_global = cls.find_class return obj.load() + + else: + class SafeUnpickler(pickle.Unpickler): def find_class(self, module, name): return allowed_module(module, name) @@ -82,7 +88,7 @@ def loads(cls, s): class ConsistentHashRing(object): - def __init__(self, nodes, replica_count=100, hash_type='carbon_ch'): + def __init__(self, nodes, replica_count=100, hash_type="carbon_ch"): self.ring = [] self.ring_len = len(self.ring) self.nodes = set() @@ -93,8 +99,8 @@ def __init__(self, nodes, replica_count=100, hash_type='carbon_ch'): self.add_node(node) def compute_ring_position(self, key): - if self.hash_type == 'fnv1a_ch': - big_hash = '{0:x}'.format(int(fnv32a(str(key)))) + if self.hash_type == "fnv1a_ch": + big_hash = "{0:x}".format(int(fnv32a(str(key)))) small_hash = int(big_hash[:4], 16) ^ int(big_hash[4:], 16) else: big_hash = md5(str(key).encode()).hexdigest() @@ -105,7 +111,7 @@ def add_node(self, key): self.nodes.add(key) self.nodes_len = len(self.nodes) for i in range(self.replica_count): - if self.hash_type == 'fnv1a_ch': + if self.hash_type == "fnv1a_ch": replica_key = "%d-%s" % (i, key[1]) else: replica_key = "%s:%d" % (key, i) @@ -145,16 +151,23 @@ def get_nodes(self, key): class CarbonLinkPool(object): - def __init__(self, hosts, timeout=1, retry_delay=15, - carbon_prefix='carbon', replication_factor=1, - hashing_keyfunc=lambda x: x, hashing_type='carbon_ch'): + def __init__( + self, + hosts, + timeout=1, + retry_delay=15, + carbon_prefix="carbon", + replication_factor=1, + hashing_keyfunc=lambda x: x, + hashing_type="carbon_ch", + ): self.carbon_prefix = carbon_prefix self.retry_delay = retry_delay self.hosts = [] self.ports = {} servers = set() for host in hosts: - parts = host.split(':') + parts = host.split(":") if len(parts) == 2: parts.append(None) server, port, instance = parts @@ -165,11 +178,14 @@ def __init__(self, hosts, timeout=1, retry_delay=15, self.timeout = float(timeout) if len(servers) < replication_factor: raise Exception( - "replication_factor=%d cannot exceed servers=%d" % ( - replication_factor, len(servers))) + "replication_factor=%d cannot exceed servers=%d" + % (replication_factor, len(servers)) + ) self.replication_factor = replication_factor - self.hash_ring = ConsistentHashRing(self.hosts, hash_type=hashing_type) + self.hash_ring = ConsistentHashRing( + self.hosts, hash_type=hashing_type + ) self.keyfunc = hashing_keyfunc self.connections = {} self.last_failure = {} @@ -224,34 +240,43 @@ def get_connection(self, host): def query(self, metric): if not self.hosts: return [] - request = dict(type='cache-query', metric=metric) + request = dict(type="cache-query", metric=metric) results = self.send_request(request) - logger.debug("carbonlink request returned", metric=metric, - datapoints=len(results['datapoints'])) - return results['datapoints'] + logger.debug( + "carbonlink request returned", + metric=metric, + datapoints=len(results["datapoints"]), + ) + return results["datapoints"] def get_metadata(self, metric, key): - request = dict(type='get-metadata', metric=metric, key=key) + request = dict(type="get-metadata", metric=metric, key=key) results = self.send_request(request) - logger.debug("carbonlink get-metadata request received", - metric=metric, key=key) - return results['value'] + logger.debug( + "carbonlink get-metadata request received", metric=metric, key=key + ) + return results["value"] def set_metadata(self, metric, key, value): - request = dict(type='set-metadata', metric=metric, - key=key, value=value) + request = dict( + type="set-metadata", metric=metric, key=key, value=value + ) results = self.send_request(request) - logger.debug("carbonlink set-metadata request received", - metric=metric, key=key, value=value) + logger.debug( + "carbonlink set-metadata request received", + metric=metric, + key=key, + value=value, + ) return results def send_request(self, request): - metric = request['metric'] + metric = request["metric"] serialized_request = pickle.dumps(request, protocol=2) len_prefix = struct.pack("!L", len(serialized_request)) request_packet = len_prefix + serialized_request result = {} - result.setdefault('datapoints', []) + result.setdefault("datapoints", []) if metric.startswith(self.carbon_prefix): return self.send_request_to_all(request) @@ -267,20 +292,21 @@ def send_request(self, request): logger.info("carbonlink exception", exc_info=True, host=str(host)) else: self.connections[host].add(conn) - if 'error' in result: - logger.info("carbonlink error", error=result['error']) - raise CarbonLinkRequestError(result['error']) - logger.debug("carbonlink finished receiving", - metric=metric, host=host) + if "error" in result: + logger.info("carbonlink error", error=result["error"]) + raise CarbonLinkRequestError(result["error"]) + logger.debug( + "carbonlink finished receiving", metric=metric, host=host + ) return result def send_request_to_all(self, request): - metric = request['metric'] + metric = request["metric"] serialized_request = pickle.dumps(request, protocol=2) len_prefix = struct.pack("!L", len(serialized_request)) request_packet = len_prefix + serialized_request results = {} - results.setdefault('datapoints', []) + results.setdefault("datapoints", []) for host in self.hosts: conn = self.get_connection(host) @@ -290,18 +316,23 @@ def send_request_to_all(self, request): result = self.recv_response(conn) except Exception: self.last_failure[host] = time.time() - logger.info("carbonlink exception", exc_info=True, - host=str(host)) + logger.info( + "carbonlink exception", exc_info=True, host=str(host) + ) else: self.connections[host].add(conn) - if 'error' in result: - logger.info("carbonlink error", - host=str(host), error=result['error']) + if "error" in result: + logger.info( + "carbonlink error", + host=str(host), + error=result["error"], + ) else: - if len(result['datapoints']) > 1: - results['datapoints'].extend(result['datapoints']) - logger.debug("carbonlink finished receiving", - metric=metric, host=str(host)) + if len(result["datapoints"]) > 1: + results["datapoints"].extend(result["datapoints"]) + logger.debug( + "carbonlink finished receiving", metric=metric, host=str(host) + ) return results def recv_response(self, conn): @@ -317,7 +348,7 @@ class CarbonLinkRequestError(Exception): # Socket helper functions def recv_exactly(conn, num_bytes): - buf = b'' + buf = b"" while len(buf) < num_bytes: data = conn.recv(num_bytes - len(buf)) if not data: diff --git a/graphite_api/config.py b/graphite_api/config.py index 31ea4e2..afa9cd3 100644 --- a/graphite_api/config.py +++ b/graphite_api/config.py @@ -8,8 +8,11 @@ import structlog import yaml from flask import make_response -from structlog.processors import (format_exc_info, JSONRenderer, - KeyValueRenderer) +from structlog.processors import ( + format_exc_info, + JSONRenderer, + KeyValueRenderer, +) from tzlocal import get_localzone from . import DEBUG @@ -24,30 +27,41 @@ logger = structlog.get_logger() default_conf = { - 'search_index': '/srv/graphite/index', - 'finders': [ - 'graphite_api.finders.whisper.WhisperFinder', + "search_index": "/srv/graphite/index", + "finders": ["graphite_api.finders.whisper.WhisperFinder"], + "functions": [ + "graphite_api.functions.SeriesFunctions", + "graphite_api.functions.PieFunctions", ], - 'functions': [ - 'graphite_api.functions.SeriesFunctions', - 'graphite_api.functions.PieFunctions', - ], - 'whisper': { - 'directories': [ - '/srv/graphite/whisper', - ], - }, - 'time_zone': get_localzone().zone, + "whisper": {"directories": ["/srv/graphite/whisper"]}, + "time_zone": get_localzone().zone, } -if default_conf['time_zone'] == 'local': # tzlocal didn't find anything - default_conf['time_zone'] = 'UTC' +if default_conf["time_zone"] == "local": # tzlocal didn't find anything + default_conf["time_zone"] = "UTC" # attributes of a classical log record -NON_EXTRA = set(['module', 'filename', 'levelno', 'exc_text', 'pathname', - 'lineno', 'msg', 'funcName', 'relativeCreated', - 'levelname', 'msecs', 'threadName', 'name', 'created', - 'process', 'processName', 'thread']) +NON_EXTRA = set( + [ + "module", + "filename", + "levelno", + "exc_text", + "pathname", + "lineno", + "msg", + "funcName", + "relativeCreated", + "levelname", + "msecs", + "threadName", + "name", + "created", + "process", + "processName", + "thread", + ] +) class StructlogFormatter(logging.Formatter): @@ -55,36 +69,46 @@ def __init__(self, *args, **kwargs): self._bound = structlog.BoundLoggerBase(None, processors, {}) def format(self, record): - if not record.name.startswith('graphite_api'): - kw = dict(((k, v) for k, v in record.__dict__.items() - if k not in NON_EXTRA)) - kw['logger'] = record.name + if not record.name.startswith("graphite_api"): + kw = dict( + ( + (k, v) + for k, v in record.__dict__.items() + if k not in NON_EXTRA + ) + ) + kw["logger"] = record.name return self._bound._process_event( - record.levelname.lower(), record.getMessage(), kw)[0] + record.levelname.lower(), record.getMessage(), kw + )[0] return record.getMessage() def load_by_path(path): - module, klass = path.rsplit('.', 1) + module, klass = path.rsplit(".", 1) finder = import_module(module) return getattr(finder, klass) def error_handler(e): - return make_response(traceback.format_exc(), 500, - {'Content-Type': 'text/plain'}) + return make_response( + traceback.format_exc(), 500, {"Content-Type": "text/plain"} + ) def configure(app): - config_file = os.environ.get('GRAPHITE_API_CONFIG', - '/etc/graphite-api.yaml') + config_file = os.environ.get( + "GRAPHITE_API_CONFIG", "/etc/graphite-api.yaml" + ) if os.path.exists(config_file): with open(config_file) as f: config = yaml.safe_load(f) - config['path'] = config_file + config["path"] = config_file else: - warnings.warn("Unable to find configuration file at {0}, using " - "default config.".format(config_file)) + warnings.warn( + "Unable to find configuration file at {0}, using " + "default config.".format(config_file) + ) config = {} configure_logging(config) @@ -93,95 +117,107 @@ def configure(app): config.setdefault(key, value) app.statsd = None - if 'statsd' in config: + if "statsd" in config: try: from statsd import StatsClient except ImportError: - warnings.warn("'statsd' is provided in the configuration but " - "the statsd client is not installed. Please `pip " - "install statsd`.") + warnings.warn( + "'statsd' is provided in the configuration but " + "the statsd client is not installed. Please `pip " + "install statsd`." + ) else: - c = config['statsd'] - app.statsd = StatsClient(c['host'], c.get('port', 8125)) + c = config["statsd"] + app.statsd = StatsClient(c["host"], c.get("port", 8125)) app.cache = None - if 'cache' in config: + if "cache" in config: try: from flask.ext.cache import Cache except ImportError: - warnings.warn("'cache' is provided in the configuration but " - "Flask-Cache is not installed. Please `pip install " - "Flask-Cache`.") + warnings.warn( + "'cache' is provided in the configuration but " + "Flask-Cache is not installed. Please `pip install " + "Flask-Cache`." + ) else: - cache_conf = {'CACHE_DEFAULT_TIMEOUT': 60, - 'CACHE_KEY_PREFIX': 'graphite-api:'} - for key, value in config['cache'].items(): - cache_conf['CACHE_{0}'.format(key.upper())] = value + cache_conf = { + "CACHE_DEFAULT_TIMEOUT": 60, + "CACHE_KEY_PREFIX": "graphite-api:", + } + for key, value in config["cache"].items(): + cache_conf["CACHE_{0}".format(key.upper())] = value app.cache = Cache(app, config=cache_conf) - loaded_config = {'functions': {}} - for functions in config['functions']: - loaded_config['functions'].update(load_by_path(functions)) + loaded_config = {"functions": {}} + for functions in config["functions"]: + loaded_config["functions"].update(load_by_path(functions)) - if 'carbon' in config: - if 'hashing_keyfunc' in config['carbon']: - config['carbon']['hashing_keyfunc'] = load_by_path( - config['carbon']['hashing_keyfunc']) + if "carbon" in config: + if "hashing_keyfunc" in config["carbon"]: + config["carbon"]["hashing_keyfunc"] = load_by_path( + config["carbon"]["hashing_keyfunc"] + ) else: - config['carbon']['hashing_keyfunc'] = lambda x: x - loaded_config['carbon'] = config.get('carbon', None) + config["carbon"]["hashing_keyfunc"] = lambda x: x + loaded_config["carbon"] = config.get("carbon", None) finders = [] - for finder in config['finders']: + for finder in config["finders"]: finders.append(load_by_path(finder)(config)) - loaded_config['store'] = Store(finders) - app.config['GRAPHITE'] = loaded_config - app.config['TIME_ZONE'] = config['time_zone'] - logger.info("configured timezone", timezone=app.config['TIME_ZONE']) + loaded_config["store"] = Store(finders) + app.config["GRAPHITE"] = loaded_config + app.config["TIME_ZONE"] = config["time_zone"] + logger.info("configured timezone", timezone=app.config["TIME_ZONE"]) - if 'sentry_dsn' in config: + if "sentry_dsn" in config: try: from raven.contrib.flask import Sentry except ImportError: - warnings.warn("'sentry_dsn' is provided in the configuration but " - "the sentry client is not installed. Please `pip " - "install raven[flask]`.") + warnings.warn( + "'sentry_dsn' is provided in the configuration but " + "the sentry client is not installed. Please `pip " + "install raven[flask]`." + ) else: - Sentry(app, dsn=config['sentry_dsn']) + Sentry(app, dsn=config["sentry_dsn"]) - app.wsgi_app = TrailingSlash(CORS(app.wsgi_app, - config.get('allowed_origins'))) - if config.get('render_errors', True): + app.wsgi_app = TrailingSlash( + CORS(app.wsgi_app, config.get("allowed_origins")) + ) + if config.get("render_errors", True): app.errorhandler(500)(error_handler) def configure_logging(config): - structlog.configure(processors=processors, - logger_factory=structlog.stdlib.LoggerFactory(), - wrapper_class=structlog.stdlib.BoundLogger, - cache_logger_on_first_use=True) - config.setdefault('logging', {}) - config['logging'].setdefault('version', 1) - config['logging'].setdefault('handlers', {}) - config['logging'].setdefault('formatters', {}) - config['logging'].setdefault('loggers', {}) - config['logging']['handlers'].setdefault('raw', { - 'level': 'DEBUG', - 'class': 'logging.StreamHandler', - 'formatter': 'raw', - }) - config['logging']['loggers'].setdefault('root', { - 'handlers': ['raw'], - 'level': 'DEBUG', - 'propagate': False, - }) - config['logging']['loggers'].setdefault('graphite_api', { - 'handlers': ['raw'], - 'level': 'DEBUG', - }) - config['logging']['formatters']['raw'] = {'()': StructlogFormatter} - dictConfig(config['logging']) - if 'path' in config: - logger.info("loading configuration", path=config['path']) + structlog.configure( + processors=processors, + logger_factory=structlog.stdlib.LoggerFactory(), + wrapper_class=structlog.stdlib.BoundLogger, + cache_logger_on_first_use=True, + ) + config.setdefault("logging", {}) + config["logging"].setdefault("version", 1) + config["logging"].setdefault("handlers", {}) + config["logging"].setdefault("formatters", {}) + config["logging"].setdefault("loggers", {}) + config["logging"]["handlers"].setdefault( + "raw", + { + "level": "DEBUG", + "class": "logging.StreamHandler", + "formatter": "raw", + }, + ) + config["logging"]["loggers"].setdefault( + "root", {"handlers": ["raw"], "level": "DEBUG", "propagate": False} + ) + config["logging"]["loggers"].setdefault( + "graphite_api", {"handlers": ["raw"], "level": "DEBUG"} + ) + config["logging"]["formatters"]["raw"] = {"()": StructlogFormatter} + dictConfig(config["logging"]) + if "path" in config: + logger.info("loading configuration", path=config["path"]) else: logger.info("loading default configuration") diff --git a/graphite_api/encoders.py b/graphite_api/encoders.py index e93d06b..162d630 100644 --- a/graphite_api/encoders.py +++ b/graphite_api/encoders.py @@ -5,14 +5,15 @@ class JSONEncoder(json.JSONEncoder): """ JSONEncoder subclass that knows how to encode generators. """ + def default(self, o): - if hasattr(o, 'tolist'): + if hasattr(o, "tolist"): return o.tolist() - elif hasattr(o, '__getitem__'): + elif hasattr(o, "__getitem__"): try: return dict(o) except TypeError: pass - elif hasattr(o, '__iter__'): + elif hasattr(o, "__iter__"): return [i for i in o] return super(JSONEncoder, self).default(o) diff --git a/graphite_api/evaluator.py b/graphite_api/evaluator.py index 012841e..2da6b01 100644 --- a/graphite_api/evaluator.py +++ b/graphite_api/evaluator.py @@ -27,35 +27,47 @@ def pathsFromTokens(requestContext, tokens, replacements=None): for i, arg in enumerate(tokens.template.args): if arg.string: arglist[str(i + 1)] = arg.string[1:-1] - if 'template' in requestContext: - arglist.update(requestContext['template']) - iters.append(pathsFromTokens(requestContext, tokens.template, arglist)) + if "template" in requestContext: + arglist.update(requestContext["template"]) + iters.append( + pathsFromTokens(requestContext, tokens.template, arglist) + ) elif tokens.expression: - iters.append(pathsFromTokens(requestContext, tokens.expression, - replacements)) + iters.append( + pathsFromTokens(requestContext, tokens.expression, replacements) + ) elif tokens.pathExpression: expression = tokens.pathExpression if replacements: for name in replacements: val = replacements[name] - expression = expression.replace('$'+name, str(val)) + expression = expression.replace("$" + name, str(val)) iters.append([expression]) elif tokens.call: - if tokens.call.funcname == 'template': + if tokens.call.funcname == "template": # if template propagates down here, it means the grammar didn't # match the invocation as tokens.template. this generally happens # if you try to pass non-numeric/string args - raise ValueError("invalid template() syntax, only string/numeric " - "arguments are allowed") - - iters.extend([pathsFromTokens(requestContext, arg, replacements) - for arg in tokens.call.args]) - iters.extend([pathsFromTokens(requestContext, kwarg.args[0], - replacements) - for kwarg in tokens.call.kwargs]) + raise ValueError( + "invalid template() syntax, only string/numeric " + "arguments are allowed" + ) + + iters.extend( + [ + pathsFromTokens(requestContext, arg, replacements) + for arg in tokens.call.args + ] + ) + iters.extend( + [ + pathsFromTokens(requestContext, kwarg.args[0], replacements) + for kwarg in tokens.call.kwargs + ] + ) for path in itertools.chain(*iters): yield path @@ -75,7 +87,9 @@ def evaluateTarget(requestContext, target, data_store=None): return result -def evaluateTokens(requestContext, tokens, data_store=None, replacements=None): +def evaluateTokens( + requestContext, tokens, data_store=None, replacements=None +): if data_store is None: paths = list(pathsFromTokens(requestContext, tokens)) data_store = fetchData(requestContext, paths) @@ -83,57 +97,77 @@ def evaluateTokens(requestContext, tokens, data_store=None, replacements=None): if tokens.template: arglist = dict() if tokens.template.kwargs: - args = [(kwarg.argname, evaluateTokens(requestContext, - kwarg.args[0], - data_store)) - for kwarg in tokens.template.kwargs] + args = [ + ( + kwarg.argname, + evaluateTokens(requestContext, kwarg.args[0], data_store), + ) + for kwarg in tokens.template.kwargs + ] arglist.update(dict(args)) if tokens.template.args: - args = [(str(i + 1), evaluateTokens(requestContext, arg, - data_store)) - for i, arg in enumerate(tokens.template.args)] + args = [ + (str(i + 1), evaluateTokens(requestContext, arg, data_store)) + for i, arg in enumerate(tokens.template.args) + ] arglist.update(dict(args)) - if 'template' in requestContext: - arglist.update(requestContext['template']) - return evaluateTokens(requestContext, tokens.template, data_store, - arglist) + if "template" in requestContext: + arglist.update(requestContext["template"]) + return evaluateTokens( + requestContext, tokens.template, data_store, arglist + ) elif tokens.expression: - return evaluateTokens(requestContext, tokens.expression, data_store, - replacements) + return evaluateTokens( + requestContext, tokens.expression, data_store, replacements + ) elif tokens.pathExpression: expression = tokens.pathExpression if replacements: for name in replacements: val = replacements[name] - if expression == '$'+name: + if expression == "$" + name: if not isinstance(val, six.string_types): return val - elif re.match('^-?[\d.]+$', val): + elif re.match("^-?[\d.]+$", val): return float(val) else: return val else: - expression = expression.replace('$'+name, str(val)) + expression = expression.replace("$" + name, str(val)) return data_store.get_series_list(expression) elif tokens.call: - if tokens.call.funcname == 'template': + if tokens.call.funcname == "template": # if template propagates down here, it means the grammar didn't # match the invocation as tokens.template. this generally happens # if you try to pass non-numeric/string args - raise ValueError("invalid template() syntax, only string/numeric " - "arguments are allowed") + raise ValueError( + "invalid template() syntax, only string/numeric " + "arguments are allowed" + ) func = app.functions[tokens.call.funcname] - args = [evaluateTokens(requestContext, arg, data_store, replacements) - for arg in tokens.call.args] - requestContext['args'] = tokens.call.args - kwargs = dict([(kwarg.argname, - evaluateTokens(requestContext, kwarg.args[0], - data_store, replacements)) - for kwarg in tokens.call.kwargs]) + args = [ + evaluateTokens(requestContext, arg, data_store, replacements) + for arg in tokens.call.args + ] + requestContext["args"] = tokens.call.args + kwargs = dict( + [ + ( + kwarg.argname, + evaluateTokens( + requestContext, + kwarg.args[0], + data_store, + replacements, + ), + ) + for kwarg in tokens.call.kwargs + ] + ) ret = func(requestContext, *args, **kwargs) return ret @@ -149,9 +183,10 @@ def evaluateTokens(requestContext, tokens, data_store=None, replacements=None): return tokens.string[1:-1] elif tokens.boolean: - return tokens.boolean[0] == 'true' + return tokens.boolean[0] == "true" else: raise ValueError("unknown token in target evaluator") + from .app import app # noqa diff --git a/graphite_api/finders/__init__.py b/graphite_api/finders/__init__.py index 83b606d..88a8d5b 100644 --- a/graphite_api/finders/__init__.py +++ b/graphite_api/finders/__init__.py @@ -2,17 +2,17 @@ import os.path import re -EXPAND_BRACES_RE = re.compile(r'.*(\{.*?[^\\]?\})') +EXPAND_BRACES_RE = re.compile(r".*(\{.*?[^\\]?\})") def get_real_metric_path(absolute_path, metric_path): # Support symbolic links (real_metric_path ensures proper cache queries) real_fs_path = os.path.realpath(absolute_path) if absolute_path != real_fs_path: - relative_fs_path = metric_path.replace('.', os.sep) - abs_fs_path = os.path.dirname(absolute_path[:-len(relative_fs_path)]) + relative_fs_path = metric_path.replace(".", os.sep) + abs_fs_path = os.path.dirname(absolute_path[: -len(relative_fs_path)]) base_fs_path = os.path.realpath(abs_fs_path) - relative_real_fs_path = real_fs_path[len(base_fs_path):].lstrip('/') + relative_real_fs_path = real_fs_path[len(base_fs_path) :].lstrip("/") return fs_to_metric(relative_real_fs_path) return metric_path @@ -21,7 +21,7 @@ def get_real_metric_path(absolute_path, metric_path): def fs_to_metric(path): dirpath = os.path.dirname(path) filename = os.path.basename(path) - return os.path.join(dirpath, filename.split('.')[0]).replace(os.sep, '.') + return os.path.join(dirpath, filename.split(".")[0]).replace(os.sep, ".") def _deduplicate(entries): @@ -34,10 +34,10 @@ def _deduplicate(entries): def extract_variants(pattern): """Extract the pattern variants (ie. {foo,bar}baz = foobaz or barbaz).""" - v1, v2 = pattern.find('{'), pattern.find('}') + v1, v2 = pattern.find("{"), pattern.find("}") if v1 > -1 and v2 > v1: - variations = pattern[v1+1:v2].split(',') - variants = [pattern[:v1] + v + pattern[v2+1:] for v in variations] + variations = pattern[v1 + 1 : v2].split(",") + variants = [pattern[:v1] + v + pattern[v2 + 1 :] for v in variations] else: variants = [pattern] return list(_deduplicate(variants)) @@ -66,7 +66,7 @@ def expand_braces(pattern): # Used instead of s.strip('{}') because strip is greedy. # We want to remove only ONE leading { and ONE trailing }, if both exist def remove_outer_braces(s): - if s[0] == '{' and s[-1] == '}': + if s[0] == "{" and s[-1] == "}": return s[1:-1] return s @@ -75,13 +75,15 @@ def remove_outer_braces(s): sub = match.group(1) v1, v2 = match.span(1) if "," in sub: - for pat in sub.strip('{}').split(','): + for pat in sub.strip("{}").split(","): subpattern = pattern[:v1] + pat + pattern[v2:] res.update(expand_braces(subpattern)) else: - subpattern = pattern[:v1] + remove_outer_braces(sub) + pattern[v2:] + subpattern = ( + pattern[:v1] + remove_outer_braces(sub) + pattern[v2:] + ) res.update(expand_braces(subpattern)) else: - res.add(pattern.replace('\\}', '}')) + res.add(pattern.replace("\\}", "}")) return list(res) diff --git a/graphite_api/finders/whisper.py b/graphite_api/finders/whisper.py index 1794d40..4cef0d1 100644 --- a/graphite_api/finders/whisper.py +++ b/graphite_api/finders/whisper.py @@ -23,51 +23,59 @@ class WhisperFinder(object): def __init__(self, config): - self.directories = config['whisper']['directories'] + self.directories = config["whisper"]["directories"] self.carbonlink = None - if 'carbon' in config: - self.carbonlink = CarbonLinkPool(**config['carbon']) + if "carbon" in config: + self.carbonlink = CarbonLinkPool(**config["carbon"]) else: self.carbonlink = None def find_nodes(self, query): - logger.debug("find_nodes", finder="whisper", start=query.startTime, - end=query.endTime, pattern=query.pattern) - clean_pattern = query.pattern.replace('\\', '') - pattern_parts = clean_pattern.split('.') + logger.debug( + "find_nodes", + finder="whisper", + start=query.startTime, + end=query.endTime, + pattern=query.pattern, + ) + clean_pattern = query.pattern.replace("\\", "") + pattern_parts = clean_pattern.split(".") for root_dir in self.directories: if not os.path.isdir(root_dir): os.makedirs(root_dir) for absolute_path in self._find_paths(root_dir, pattern_parts): - if os.path.basename(absolute_path).startswith('.'): + if os.path.basename(absolute_path).startswith("."): continue - relative_path = absolute_path[len(root_dir):].lstrip(os.sep) + relative_path = absolute_path[len(root_dir) :].lstrip(os.sep) metric_path = fs_to_metric(relative_path) - real_metric_path = get_real_metric_path(absolute_path, - metric_path) + real_metric_path = get_real_metric_path( + absolute_path, metric_path + ) - metric_path_parts = metric_path.split('.') + metric_path_parts = metric_path.split(".") for field_index in find_escaped_pattern_fields(query.pattern): metric_path_parts[field_index] = pattern_parts[ - field_index].replace('\\', '') - metric_path = '.'.join(metric_path_parts) + field_index + ].replace("\\", "") + metric_path = ".".join(metric_path_parts) # Now we construct and yield an appropriate Node object if os.path.isdir(absolute_path): yield BranchNode(metric_path) elif os.path.isfile(absolute_path): - if absolute_path.endswith('.wsp'): - reader = WhisperReader(absolute_path, real_metric_path, - self.carbonlink) + if absolute_path.endswith(".wsp"): + reader = WhisperReader( + absolute_path, real_metric_path, self.carbonlink + ) yield LeafNode(metric_path, reader) - elif absolute_path.endswith('.wsp.gz'): - reader = GzippedWhisperReader(absolute_path, - real_metric_path, - self.carbonlink) + elif absolute_path.endswith(".wsp.gz"): + reader = GzippedWhisperReader( + absolute_path, real_metric_path, self.carbonlink + ) yield LeafNode(metric_path, reader) def _find_paths(self, current_dir, patterns): @@ -88,13 +96,16 @@ def _find_paths(self, current_dir, patterns): if using_globstar: matching_subdirs = map(lambda x: x[0], walk(current_dir)) else: - subdirs = [e for e in entries - if os.path.isdir(os.path.join(current_dir, e))] + subdirs = [ + e + for e in entries + if os.path.isdir(os.path.join(current_dir, e)) + ] matching_subdirs = match_entries(subdirs, pattern) # For terminal globstar, add a pattern for all files in subdirs if using_globstar and not patterns: - patterns = ['*'] + patterns = ["*"] if patterns: # we've still got more directories to traverse for subdir in matching_subdirs: @@ -104,10 +115,13 @@ def _find_paths(self, current_dir, patterns): else: # we've got the last pattern if not has_wildcard: - entries = [pattern + '.wsp', pattern + '.wsp.gz'] - files = [e for e in entries - if os.path.isfile(os.path.join(current_dir, e))] - matching_files = match_entries(files, pattern + '.*') + entries = [pattern + ".wsp", pattern + ".wsp.gz"] + files = [ + e + for e in entries + if os.path.isfile(os.path.join(current_dir, e)) + ] + matching_files = match_entries(files, pattern + ".*") for _basename in matching_files + matching_subdirs: yield os.path.join(current_dir, _basename) @@ -115,7 +129,7 @@ def _find_paths(self, current_dir, patterns): class WhisperReader(object): - __slots__ = ('fs_path', 'real_metric_path', 'carbonlink') + __slots__ = ("fs_path", "real_metric_path", "carbonlink") def __init__(self, fs_path, real_metric_path, carbonlink=None): self.fs_path = fs_path @@ -123,14 +137,19 @@ def __init__(self, fs_path, real_metric_path, carbonlink=None): self.carbonlink = carbonlink def get_intervals(self): - start = time.time() - whisper.info(self.fs_path)['maxRetention'] + start = time.time() - whisper.info(self.fs_path)["maxRetention"] end = max(stat(self.fs_path).st_mtime, start) return IntervalSet([Interval(start, end)]) def fetch(self, startTime, endTime): # noqa - logger.debug("fetch", reader="whisper", path=self.fs_path, - metric_path=self.real_metric_path, - start=startTime, end=endTime) + logger.debug( + "fetch", + reader="whisper", + path=self.fs_path, + metric_path=self.real_metric_path, + start=startTime, + end=endTime, + ) data = whisper.fetch(self.fs_path, startTime, endTime) if not data: return None @@ -155,21 +174,28 @@ def fetch(self, startTime, endTime): # noqa class GzippedWhisperReader(WhisperReader): def get_intervals(self): - fh = gzip.GzipFile(self.fs_path, 'rb') + fh = gzip.GzipFile(self.fs_path, "rb") try: - info = getattr(whisper, '__readHeader')(fh) # evil, but necessary. + info = getattr(whisper, "__readHeader")( + fh + ) # evil, but necessary. finally: fh.close() - start = time.time() - info['maxRetention'] + start = time.time() - info["maxRetention"] end = max(stat(self.fs_path).st_mtime, start) return IntervalSet([Interval(start, end)]) def fetch(self, startTime, endTime): - logger.debug("fetch", reader="gzip_whisper", path=self.fs_path, - metric_path=self.real_metric_path, - start=startTime, end=endTime) - fh = gzip.GzipFile(self.fs_path, 'rb') + logger.debug( + "fetch", + reader="gzip_whisper", + path=self.fs_path, + metric_path=self.real_metric_path, + start=startTime, + end=endTime, + ) + fh = gzip.GzipFile(self.fs_path, "rb") try: return whisper.file_fetch(fh, startTime, endTime) finally: @@ -177,16 +203,16 @@ def fetch(self, startTime, endTime): def find_escaped_pattern_fields(pattern_string): - pattern_parts = pattern_string.split('.') + pattern_parts = pattern_string.split(".") for index, part in enumerate(pattern_parts): if is_escaped_pattern(part): yield index def is_escaped_pattern(s): - for symbol in '*?[{': + for symbol in "*?[{": i = s.find(symbol) if i > 0: - if s[i-1] == '\\': + if s[i - 1] == "\\": return True return False diff --git a/graphite_api/functions.py b/graphite_api/functions.py index 9f072e4..745b506 100644 --- a/graphite_api/functions.py +++ b/graphite_api/functions.py @@ -31,8 +31,8 @@ from .render.grammar import grammar from .utils import epoch, to_seconds -NAN = float('NaN') -INF = float('inf') +NAN = float("NaN") +INF = float("inf") MINUTE = 60 HOUR = MINUTE * 60 DAY = HOUR * 24 @@ -55,6 +55,7 @@ def inner(values): if not vals: return return f(vals) + return inner @@ -124,7 +125,7 @@ def safeStdDev(a): sum = 0 for val in not_none(a): sum = sum + (val - avg) * (val - avg) - return math.sqrt(sum/ln) + return math.sqrt(sum / ln) def safeLast(values): @@ -164,7 +165,7 @@ def lcm(a, b): def normalize(seriesLists): if not seriesLists: return [] - seriesList = reduce(lambda L1, L2: L1+L2, seriesLists) + seriesList = reduce(lambda L1, L2: L1 + L2, seriesLists) step = reduce(lcm, [s.step for s in seriesList]) for s in seriesList: s.consolidate(step // s.step) @@ -175,11 +176,13 @@ def normalize(seriesLists): def matchSeries(seriesList1, seriesList2): - assert len(seriesList2) == len(seriesList1), ("The number of series in " - "each argument must be the " - "same") - return zip_longest(sorted(seriesList1, key=lambda x: x.name), - sorted(seriesList2, key=lambda x: x.name)) + assert len(seriesList2) == len(seriesList1), ( + "The number of series in " "each argument must be the " "same" + ) + return zip_longest( + sorted(seriesList1, key=lambda x: x.name), + sorted(seriesList2, key=lambda x: x.name), + ) def formatPathExpressions(seriesList): @@ -187,7 +190,8 @@ def formatPathExpressions(seriesList): Returns a comma-separated list of unique path expressions. """ pathExpressions = sorted(set([s.pathExpression for s in seriesList])) - return ','.join(pathExpressions) + return ",".join(pathExpressions) + # Series Functions @@ -239,12 +243,19 @@ def sumSeriesWithWildcards(requestContext, seriesList, *positions): newNames = list() for series in seriesList: - newname = '.'.join(map(lambda x: x[1], - filter(lambda i: i[0] not in positions, - enumerate(series.name.split('.'))))) + newname = ".".join( + map( + lambda x: x[1], + filter( + lambda i: i[0] not in positions, + enumerate(series.name.split(".")), + ), + ) + ) if newname in newSeries: - newSeries[newname] = sumSeries(requestContext, - (series, newSeries[newname]))[0] + newSeries[newname] = sumSeries( + requestContext, (series, newSeries[newname]) + )[0] else: newSeries[newname] = series newNames.append(newname) @@ -270,9 +281,15 @@ def averageSeriesWithWildcards(requestContext, seriesList, *positions): """ matchedList = defaultdict(list) for series in seriesList: - newname = '.'.join(map(lambda x: x[1], - filter(lambda i: i[0] not in positions, - enumerate(series.name.split('.'))))) + newname = ".".join( + map( + lambda x: x[1], + filter( + lambda i: i[0] not in positions, + enumerate(series.name.split(".")), + ), + ) + ) matchedList[newname].append(series) result = [] for name in matchedList: @@ -303,14 +320,20 @@ def multiplySeriesWithWildcards(requestContext, seriesList, *position): newNames = [] for series in seriesList: - new_name = ".".join(map(lambda x: x[1], - filter(lambda i: i[0] not in positions, - enumerate(series.name.split('.'))))) + new_name = ".".join( + map( + lambda x: x[1], + filter( + lambda i: i[0] not in positions, + enumerate(series.name.split(".")), + ), + ) + ) if new_name in newSeries: - [newSeries[new_name]] = multiplySeries(requestContext, - (newSeries[new_name], - series)) + [newSeries[new_name]] = multiplySeries( + requestContext, (newSeries[new_name], series) + ) else: newSeries[new_name] = series newNames.append(new_name) @@ -363,8 +386,10 @@ def averageSeries(requestContext, *seriesLists): return [] seriesList, start, end, step = normalize(seriesLists) name = "averageSeries(%s)" % formatPathExpressions(seriesList) - values = (safeDiv(safeSum(row), safeLen(row)) - for row in zip_longest(*seriesList)) + values = ( + safeDiv(safeSum(row), safeLen(row)) + for row in zip_longest(*seriesList) + ) series = TimeSeries(name, start, end, step, values) series.pathExpression = name return [series] @@ -445,8 +470,9 @@ def rangeOfSeries(requestContext, *seriesLists): return [] seriesList, start, end, step = normalize(seriesLists) name = "rangeOfSeries(%s)" % formatPathExpressions(seriesList) - values = (safeSubtract(max(row), - min(row)) for row in zip_longest(*seriesList)) + values = ( + safeSubtract(max(row), min(row)) for row in zip_longest(*seriesList) + ) series = TimeSeries(name, start, end, step, values) series.pathExpression = name return [series] @@ -461,14 +487,17 @@ def percentileOfSeries(requestContext, seriesList, n, interpolate=False): """ if n <= 0: raise ValueError( - 'The requested percent is required to be greater than 0') + "The requested percent is required to be greater than 0" + ) if not seriesList: return [] - name = 'percentileOfSeries(%s,%g)' % (seriesList[0].pathExpression, n) + name = "percentileOfSeries(%s,%g)" % (seriesList[0].pathExpression, n) start, end, step = normalize([seriesList])[1:] - values = [_getPercentile(row, n, interpolate) - for row in zip_longest(*seriesList)] + values = [ + _getPercentile(row, n, interpolate) + for row in zip_longest(*seriesList) + ] resultSeries = TimeSeries(name, start, end, step, values) resultSeries.pathExpression = name return [resultSeries] @@ -580,7 +609,7 @@ def changed(requestContext, seriesList): &target=changed(Server01.connections.handled) """ for series in seriesList: - series.name = series.pathExpression = 'changed(%s)' % series.name + series.name = series.pathExpression = "changed(%s)" % series.name previous = None for index, value in enumerate(series): if previous is None: @@ -623,10 +652,12 @@ def asPercent(requestContext, seriesList, total=None): totalText = "sumSeries(%s)" % formatPathExpressions(seriesList) elif type(total) is list: if len(total) != 1 and len(total) != len(seriesList): - raise ValueError("asPercent second argument must be missing, a " - "single digit, reference exactly 1 series or " - "reference the same number of series as the " - "first argument") + raise ValueError( + "asPercent second argument must be missing, a " + "single digit, reference exactly 1 series or " + "reference the same number of series as the " + "first argument" + ) if len(total) == 1: normalize([seriesList, total]) @@ -641,20 +672,26 @@ def asPercent(requestContext, seriesList, total=None): for series1, series2 in matchSeries(seriesList, total): name = "asPercent(%s,%s)" % (series1.name, series2.name) (seriesList, start, end, step) = normalize([(series1, series2)]) - resultValues = [safeMul(safeDiv(v1, v2), 100.0) - for v1, v2 in zip_longest(series1, series2)] + resultValues = [ + safeMul(safeDiv(v1, v2), 100.0) + for v1, v2 in zip_longest(series1, series2) + ] resultSeries = TimeSeries(name, start, end, step, resultValues) resultSeries.pathExpression = name resultList.append(resultSeries) else: for series in seriesList: - resultValues = [safeMul(safeDiv(val, totalVal), 100.0) - for val, totalVal in zip_longest(series, - totalValues)] - name = "asPercent(%s,%s)" % (series.name, totalText or - series.pathExpression) - resultSeries = TimeSeries(name, series.start, series.end, - series.step, resultValues) + resultValues = [ + safeMul(safeDiv(val, totalVal), 100.0) + for val, totalVal in zip_longest(series, totalValues) + ] + name = "asPercent(%s,%s)" % ( + series.name, + totalText or series.pathExpression, + ) + resultSeries = TimeSeries( + name, series.start, series.end, series.step, resultValues + ) resultSeries.pathExpression = name resultList.append(resultSeries) @@ -668,15 +705,20 @@ def divideSeriesLists(requestContext, dividendSeriesList, divisorSeriesList): """ if len(dividendSeriesList) != len(divisorSeriesList): - raise ValueError("dividendSeriesList and divisorSeriesList argument\ - must have equal length") + raise ValueError( + "dividendSeriesList and divisorSeriesList argument\ + must have equal length" + ) results = [] - for dividendSeries, divisorSeries in zip(dividendSeriesList, - divisorSeriesList): - name = "divideSeries(%s,%s)" % (dividendSeries.name, - divisorSeries.name) + for dividendSeries, divisorSeries in zip( + dividendSeriesList, divisorSeriesList + ): + name = "divideSeries(%s,%s)" % ( + dividendSeries.name, + divisorSeries.name, + ) bothSeries = (dividendSeries, divisorSeries) step = reduce(lcm, [s.step for s in bothSeries]) @@ -718,14 +760,17 @@ def divideSeries(requestContext, dividendSeriesList, divisorSeriesList): if len(divisorSeriesList) > 1: raise ValueError( "divideSeries second argument must reference exactly 1 series" - " (got {0})".format(len(divisorSeriesList))) + " (got {0})".format(len(divisorSeriesList)) + ) [divisorSeries] = divisorSeriesList results = [] for dividendSeries in dividendSeriesList: - name = "divideSeries(%s,%s)" % (dividendSeries.name, - divisorSeries.name) + name = "divideSeries(%s,%s)" % ( + dividendSeries.name, + divisorSeries.name, + ) bothSeries = (dividendSeries, divisorSeries) step = reduce(lcm, [s.step for s in bothSeries]) @@ -763,7 +808,7 @@ def multiplySeries(requestContext, *seriesLists): if len(seriesList) == 1: return seriesList - name = "multiplySeries(%s)" % ','.join([s.name for s in seriesList]) + name = "multiplySeries(%s)" % ",".join([s.name for s in seriesList]) product = map(lambda x: safeMul(*x), zip_longest(*seriesList)) resultSeries = TimeSeries(name, start, end, step, product) resultSeries.pathExpression = name @@ -790,38 +835,45 @@ def weightedAverage(requestContext, seriesListAvg, seriesListWeight, *nodes): sortedSeries = {} for seriesAvg, seriesWeight in zip_longest( - seriesListAvg, seriesListWeight): - key = '' + seriesListAvg, seriesListWeight + ): + key = "" for node in nodes: key += seriesAvg.name.split(".")[node] sortedSeries.setdefault(key, {}) - sortedSeries[key]['avg'] = seriesAvg + sortedSeries[key]["avg"] = seriesAvg - key = '' + key = "" for node in nodes: key += seriesWeight.name.split(".")[node] sortedSeries.setdefault(key, {}) - sortedSeries[key]['weight'] = seriesWeight + sortedSeries[key]["weight"] = seriesWeight productList = [] for key in sortedSeries: - if 'weight' not in sortedSeries[key]: + if "weight" not in sortedSeries[key]: continue - if 'avg' not in sortedSeries[key]: + if "avg" not in sortedSeries[key]: continue - seriesWeight = sortedSeries[key]['weight'] - seriesAvg = sortedSeries[key]['avg'] - - productValues = [safeMul(val1, val2) - for val1, val2 - in zip_longest(seriesAvg, seriesWeight)] - name = 'product(%s,%s)' % (seriesWeight.name, seriesAvg.name) - productSeries = TimeSeries(name, seriesAvg.start, seriesAvg.end, - seriesAvg.step, productValues) + seriesWeight = sortedSeries[key]["weight"] + seriesAvg = sortedSeries[key]["avg"] + + productValues = [ + safeMul(val1, val2) + for val1, val2 in zip_longest(seriesAvg, seriesWeight) + ] + name = "product(%s,%s)" % (seriesWeight.name, seriesAvg.name) + productSeries = TimeSeries( + name, + seriesAvg.start, + seriesAvg.end, + seriesAvg.step, + productValues, + ) productSeries.pathExpression = name productList.append(productSeries) @@ -831,14 +883,22 @@ def weightedAverage(requestContext, seriesListAvg, seriesListWeight, *nodes): [sumProducts] = sumSeries(requestContext, productList) [sumWeights] = sumSeries(requestContext, seriesListWeight) - resultValues = [safeDiv(val1, val2) - for val1, val2 in zip_longest(sumProducts, sumWeights)] + resultValues = [ + safeDiv(val1, val2) + for val1, val2 in zip_longest(sumProducts, sumWeights) + ] name = "weightedAverage(%s, %s, %s)" % ( - ','.join(sorted(set(s.pathExpression for s in seriesListAvg))), - ','.join(sorted(set(s.pathExpression for s in seriesListWeight))), - ','.join(map(str, nodes))) - resultSeries = TimeSeries(name, sumProducts.start, sumProducts.end, - sumProducts.step, resultValues) + ",".join(sorted(set(s.pathExpression for s in seriesListAvg))), + ",".join(sorted(set(s.pathExpression for s in seriesListWeight))), + ",".join(map(str, nodes)), + ) + resultSeries = TimeSeries( + name, + sumProducts.start, + sumProducts.end, + sumProducts.step, + resultValues, + ) resultSeries.pathExpression = name return resultSeries @@ -876,17 +936,18 @@ def exponentialMovingAverage(requestContext, seriesList, windowSize): # set previewSeconds and constant based on windowSize string or integer if windowInterval: previewSeconds = windowInterval - constant = (float(2) / (int(windowInterval) + 1)) + constant = float(2) / (int(windowInterval) + 1) else: previewSeconds = max([s.step for s in seriesList]) * int(windowSize) - constant = (float(2) / (int(windowSize) + 1)) + constant = float(2) / (int(windowSize) + 1) # ignore original data and pull new, including our preview # data from earlier is needed to calculate the early results newContext = requestContext.copy() - newContext['startTime'] = (requestContext['startTime'] - - timedelta(seconds=previewSeconds)) - previewList = evaluateTokens(newContext, requestContext['args'][0]) + newContext["startTime"] = requestContext["startTime"] - timedelta( + seconds=previewSeconds + ) + previewList = evaluateTokens(newContext, requestContext["args"][0]) result = [] for series in previewList: @@ -897,13 +958,22 @@ def exponentialMovingAverage(requestContext, seriesList, windowSize): if isinstance(windowSize, six.string_types): newName = 'exponentialMovingAverage(%s,"%s")' % ( - series.name, windowSize) + series.name, + windowSize, + ) else: newName = "exponentialMovingAverage(%s,%s)" % ( - series.name, windowSize) - - newSeries = TimeSeries(newName, series.start + previewSeconds, - series.end, series.step, []) + series.name, + windowSize, + ) + + newSeries = TimeSeries( + newName, + series.start + previewSeconds, + series.end, + series.step, + [], + ) newSeries.pathExpression = newName window_sum = safeSum(series[:windowPoints]) or 0 count = safeLen(series[:windowPoints]) @@ -917,8 +987,9 @@ def exponentialMovingAverage(requestContext, seriesList, windowSize): for i in range(windowPoints, len(series) - 1): if series[i] is not None: - ema = (float(constant) * float(series[i]) + - (1 - float(constant)) * float(ema)) + ema = float(constant) * float(series[i]) + ( + 1 - float(constant) + ) * float(ema) newSeries.append(round(ema, 3)) else: newSeries.append(None) @@ -959,9 +1030,10 @@ def movingMedian(requestContext, seriesList, windowSize): # ignore original data and pull new, including our preview # data from earlier is needed to calculate the early results newContext = requestContext.copy() - newContext['startTime'] = (requestContext['startTime'] - - timedelta(seconds=previewSeconds)) - previewList = evaluateTokens(newContext, requestContext['args'][0]) + newContext["startTime"] = requestContext["startTime"] - timedelta( + seconds=previewSeconds + ) + previewList = evaluateTokens(newContext, requestContext["args"][0]) result = [] for series in previewList: @@ -974,12 +1046,17 @@ def movingMedian(requestContext, seriesList, windowSize): newName = 'movingMedian(%s,"%s")' % (series.name, windowSize) else: newName = "movingMedian(%s,%s)" % (series.name, windowSize) - newSeries = TimeSeries(newName, series.start + previewSeconds, - series.end, series.step, []) + newSeries = TimeSeries( + newName, + series.start + previewSeconds, + series.end, + series.step, + [], + ) newSeries.pathExpression = newName for i in range(windowPoints, len(series)): - window = series[i - windowPoints:i] + window = series[i - windowPoints : i] nonNull = [v for v in window if v is not None] if nonNull: m_index = len(nonNull) // 2 @@ -1061,7 +1138,7 @@ def powSeries(requestContext, *seriesLists): if not seriesLists or not any(seriesLists): return [] seriesList, start, end, step = normalize(seriesLists) - name = "powSeries(%s)" % ','.join([s.name for s in seriesList]) + name = "powSeries(%s)" % ",".join([s.name for s in seriesList]) values = [] for row in zip_longest(*seriesList): first = True @@ -1218,9 +1295,10 @@ def movingAverage(requestContext, seriesList, windowSize): # ignore original data and pull new, including our preview # data from earlier is needed to calculate the early results newContext = requestContext.copy() - newContext['startTime'] = (requestContext['startTime'] - - timedelta(seconds=previewSeconds)) - previewList = evaluateTokens(newContext, requestContext['args'][0]) + newContext["startTime"] = requestContext["startTime"] - timedelta( + seconds=previewSeconds + ) + previewList = evaluateTokens(newContext, requestContext["args"][0]) result = [] for series in previewList: @@ -1233,8 +1311,13 @@ def movingAverage(requestContext, seriesList, windowSize): newName = 'movingAverage(%s,"%s")' % (series.name, windowSize) else: newName = "movingAverage(%s,%s)" % (series.name, windowSize) - newSeries = TimeSeries(newName, series.start + previewSeconds, - series.end, series.step, []) + newSeries = TimeSeries( + newName, + series.start + previewSeconds, + series.end, + series.step, + [], + ) newSeries.pathExpression = newName windowSum = safeSum(series[:windowPoints]) or 0 @@ -1285,9 +1368,10 @@ def movingSum(requestContext, seriesList, windowSize): # ignore original data and pull new, including our preview # data from earlier is needed to calculate the early results newContext = requestContext.copy() - newContext['startTime'] = (requestContext['startTime'] - - timedelta(seconds=previewSeconds)) - previewList = evaluateTokens(newContext, requestContext['args'][0]) + newContext["startTime"] = requestContext["startTime"] - timedelta( + seconds=previewSeconds + ) + previewList = evaluateTokens(newContext, requestContext["args"][0]) result = [] for series in previewList: @@ -1301,8 +1385,13 @@ def movingSum(requestContext, seriesList, windowSize): else: newName = "movingSum(%s,%s)" % (series.name, windowSize) - newSeries = TimeSeries(newName, series.start + previewSeconds, - series.end, series.step, []) + newSeries = TimeSeries( + newName, + series.start + previewSeconds, + series.end, + series.step, + [], + ) newSeries.pathExpression = newName window_sum = safeSum(series[:windowPoints]) @@ -1351,9 +1440,10 @@ def movingMin(requestContext, seriesList, windowSize): # ignore original data and pull new, including our preview # data from earlier is needed to calculate the early results newContext = requestContext.copy() - newContext['startTime'] = (requestContext['startTime'] - - timedelta(seconds=previewSeconds)) - previewList = evaluateTokens(newContext, requestContext['args'][0]) + newContext["startTime"] = requestContext["startTime"] - timedelta( + seconds=previewSeconds + ) + previewList = evaluateTokens(newContext, requestContext["args"][0]) result = [] for series in previewList: @@ -1367,11 +1457,16 @@ def movingMin(requestContext, seriesList, windowSize): else: newName = "movingMin(%s,%s)" % (series.name, windowSize) - newSeries = TimeSeries(newName, series.start + previewSeconds, - series.end, series.step, []) + newSeries = TimeSeries( + newName, + series.start + previewSeconds, + series.end, + series.step, + [], + ) newSeries.pathExpression = newName for i in range(windowPoints, len(series)): - window = series[i - windowPoints:i] + window = series[i - windowPoints : i] newSeries.append(safeMin(window)) result.append(newSeries) @@ -1411,9 +1506,10 @@ def movingMax(requestContext, seriesList, windowSize): # ignore original data and pull new, including our preview # data from earlier is needed to calculate the early results newContext = requestContext.copy() - newContext['startTime'] = (requestContext['startTime'] - - timedelta(seconds=previewSeconds)) - previewList = evaluateTokens(newContext, requestContext['args'][0]) + newContext["startTime"] = requestContext["startTime"] - timedelta( + seconds=previewSeconds + ) + previewList = evaluateTokens(newContext, requestContext["args"][0]) result = [] for series in previewList: @@ -1427,11 +1523,16 @@ def movingMax(requestContext, seriesList, windowSize): else: newName = "movingMax(%s,%s)" % (series.name, windowSize) - newSeries = TimeSeries(newName, series.start + previewSeconds, - series.end, series.step, []) + newSeries = TimeSeries( + newName, + series.start + previewSeconds, + series.end, + series.step, + [], + ) newSeries.pathExpression = newName for i in range(windowPoints, len(series)): - window = series[i - windowPoints:i] + window = series[i - windowPoints : i] newSeries.append(safeMax(window)) result.append(newSeries) @@ -1458,7 +1559,7 @@ def cumulative(requestContext, seriesList): &target=cumulative(Sales.widgets.largeBlue) """ - return consolidateBy(requestContext, seriesList, 'sum') + return consolidateBy(requestContext, seriesList, "sum") def consolidateBy(requestContext, seriesList, consolidationFunc): @@ -1486,8 +1587,10 @@ def consolidateBy(requestContext, seriesList, consolidationFunc): # datalib will throw an exception, so it's not necessary to validate # here series.consolidationFunc = consolidationFunc - series.name = 'consolidateBy(%s,"%s")' % (series.name, - series.consolidationFunc) + series.name = 'consolidateBy(%s,"%s")' % ( + series.name, + series.consolidationFunc, + ) series.pathExpression = series.name return seriesList @@ -1523,8 +1626,9 @@ def derivative(requestContext, seriesList): newValues.append(val - prev) prev = val newName = "derivative(%s)" % series.name - newSeries = TimeSeries(newName, series.start, series.end, series.step, - newValues) + newSeries = TimeSeries( + newName, series.start, series.end, series.step, newValues + ) newSeries.pathExpression = newName results.append(newSeries) return results @@ -1571,8 +1675,9 @@ def perSecond(requestContext, seriesList, maxValue=None): step = series.step prev = val newName = "perSecond(%s)" % series.name - newSeries = TimeSeries(newName, series.start, series.end, series.step, - newValues) + newSeries = TimeSeries( + newName, series.start, series.end, series.step, newValues + ) newSeries.pathExpression = newName results.append(newSeries) return results @@ -1607,8 +1712,9 @@ def delay(requestContext, seriesList, steps): newValues.append(prev.pop(0)) prev.append(val) newName = "delay(%s,%d)" % (series.name, steps) - newSeries = TimeSeries(newName, series.start, series.end, series.step, - newValues) + newSeries = TimeSeries( + newName, series.start, series.end, series.step, newValues + ) newSeries.pathExpression = newName results.append(newSeries) return results @@ -1639,8 +1745,9 @@ def integral(requestContext, seriesList): current += val newValues.append(current) newName = "integral(%s)" % series.name - newSeries = TimeSeries(newName, series.start, series.end, series.step, - newValues) + newSeries = TimeSeries( + newName, series.start, series.end, series.step, newValues + ) newSeries.pathExpression = newName results.append(newSeries) return results @@ -1663,7 +1770,7 @@ def integralByInterval(requestContext, seriesList, intervalUnit): """ intervalDuration = int(to_seconds(parseTimeOffset(intervalUnit))) - startTime = int(epoch(requestContext['startTime'])) + startTime = int(epoch(requestContext["startTime"])) results = [] for series in seriesList: newValues = [] @@ -1673,9 +1780,8 @@ def integralByInterval(requestContext, seriesList, intervalUnit): current = 0.0 for val in series: # reset integral value if crossing an interval boundary - if ( - ((currentTime - startTime) // intervalDuration) != - ((currentTime - startTime - series.step) // intervalDuration) + if ((currentTime - startTime) // intervalDuration) != ( + (currentTime - startTime - series.step) // intervalDuration ): current = 0.0 if val is None: @@ -1687,8 +1793,9 @@ def integralByInterval(requestContext, seriesList, intervalUnit): newValues.append(current) currentTime += series.step newName = "integralByInterval(%s,'%s')" % (series.name, intervalUnit) - newSeries = TimeSeries(newName, series.start, series.end, series.step, - newValues) + newSeries = TimeSeries( + newName, series.start, series.end, series.step, newValues + ) newSeries.pathExpression = newName results.append(newSeries) return results @@ -1730,15 +1837,16 @@ def nonNegativeDerivative(requestContext, seriesList, maxValue=None): prev = val newName = "nonNegativeDerivative(%s)" % series.name - newSeries = TimeSeries(newName, series.start, series.end, series.step, - newValues) + newSeries = TimeSeries( + newName, series.start, series.end, series.step, newValues + ) newSeries.pathExpression = newName results.append(newSeries) return results -def stacked(requestContext, seriesLists, stackName='__DEFAULT__'): +def stacked(requestContext, seriesLists, stackName="__DEFAULT__"): """ Takes one metric or a wildcard seriesList and change them so they are stacked. This is a way of stacking just a couple of metrics without having @@ -1753,10 +1861,10 @@ def stacked(requestContext, seriesLists, stackName='__DEFAULT__'): &target=stacked(company.server.application01.ifconfig.TXPackets, 'tx') """ - if 'totalStack' in requestContext: - totalStack = requestContext['totalStack'].get(stackName, []) + if "totalStack" in requestContext: + totalStack = requestContext["totalStack"].get(stackName, []) else: - requestContext['totalStack'] = {} + requestContext["totalStack"] = {} totalStack = [] results = [] for series in seriesLists: @@ -1772,17 +1880,18 @@ def stacked(requestContext, seriesLists, stackName='__DEFAULT__'): newValues.append(None) # Work-around for the case when legend is set - if stackName == '__DEFAULT__': + if stackName == "__DEFAULT__": newName = "stacked(%s)" % series.name else: newName = series.name - newSeries = TimeSeries(newName, series.start, series.end, series.step, - newValues) - newSeries.options['stacked'] = True + newSeries = TimeSeries( + newName, series.start, series.end, series.step, newValues + ) + newSeries.options["stacked"] = True newSeries.pathExpression = newName results.append(newSeries) - requestContext['totalStack'][stackName] = totalStack + requestContext["totalStack"][stackName] = totalStack return results @@ -1813,17 +1922,18 @@ def areaBetween(requestContext, *seriesLists): """ if len(seriesLists) == 1: [seriesLists] = seriesLists - assert len(seriesLists) == 2, ("areaBetween series argument must " - "reference *exactly* 2 series") + assert len(seriesLists) == 2, ( + "areaBetween series argument must " "reference *exactly* 2 series" + ) lower, upper = seriesLists if len(lower) == 1: [lower] = lower if len(upper) == 1: [upper] = upper - lower.options['stacked'] = True - lower.options['invisible'] = True + lower.options["stacked"] = True + lower.options["invisible"] = True - upper.options['stacked'] = True + upper.options["stacked"] = True lower.name = upper.name = "areaBetween(%s)" % upper.pathExpression return [lower, upper] @@ -1876,6 +1986,7 @@ def cactiStyle(requestContext, seriesList, system=None, units=None): &target=cactiStyle(ganglia.*.net.bytes_out,"si","b") """ + def fmt(x): if system: if units: @@ -1887,13 +1998,29 @@ def fmt(x): return "%.2f %s" % (x, units) else: return "%.2f" % x + nameLen = max([0] + [len(series.name) for series in seriesList]) - lastLen = max([0] + [len(fmt(int(safeLast(series) or 3))) - for series in seriesList]) + 3 - maxLen = max([0] + [len(fmt(int(safeMax(series) or 3))) - for series in seriesList]) + 3 - minLen = max([0] + [len(fmt(int(safeMin(series) or 3))) - for series in seriesList]) + 3 + lastLen = ( + max( + [0] + + [len(fmt(int(safeLast(series) or 3))) for series in seriesList] + ) + + 3 + ) + maxLen = ( + max( + [0] + + [len(fmt(int(safeMax(series) or 3))) for series in seriesList] + ) + + 3 + ) + minLen = ( + max( + [0] + + [len(fmt(int(safeMin(series) or 3))) for series in seriesList] + ) + + 3 + ) for series in seriesList: last = safeLast(series) maximum = safeMax(series) @@ -1913,8 +2040,15 @@ def fmt(x): minimum = fmt(float(minimum)) series.name = "%*s Current:%*s Max:%*s Min:%*s " % ( - -nameLen, series.name, -lastLen, last, - -maxLen, maximum, -minLen, minimum) + -nameLen, + series.name, + -lastLen, + last, + -maxLen, + maximum, + -minLen, + minimum, + ) return seriesList @@ -1946,8 +2080,8 @@ def aliasByNode(requestContext, seriesList, *nodes): """ for series in seriesList: pathExpression = _getFirstPathExpression(series.name) - metric_pieces = pathExpression.split('.') - series.name = '.'.join(metric_pieces[n] for n in nodes) + metric_pieces = pathExpression.split(".") + series.name = ".".join(metric_pieces[n] for n in nodes) return seriesList @@ -1977,18 +2111,18 @@ def legendValue(requestContext, seriesList, *valueTypes): """ valueFuncs = { - 'avg': lambda s: safeDiv(safeSum(s), safeLen(s)), - 'total': safeSum, - 'min': safeMin, - 'max': safeMax, - 'last': safeLast, + "avg": lambda s: safeDiv(safeSum(s), safeLen(s)), + "total": safeSum, + "min": safeMin, + "max": safeMax, + "last": safeLast, } system = None - if valueTypes[-1] in ('si', 'binary'): + if valueTypes[-1] in ("si", "binary"): system = valueTypes[-1] valueTypes = valueTypes[:-1] for valueType in valueTypes: - valueFunc = valueFuncs.get(valueType, lambda s: '(?)') + valueFunc = valueFuncs.get(valueType, lambda s: "(?)") if system is None: for series in seriesList: series.name += " (%s: %s)" % (valueType, valueFunc(series)) @@ -1998,8 +2132,11 @@ def legendValue(requestContext, seriesList, *valueTypes): formatted = None if value is not None: formatted = "%.2f%s" % format_units(value, system=system) - series.name = "%-20s%-5s%-10s" % (series.name, valueType, - formatted) + series.name = "%-20s%-5s%-10s" % ( + series.name, + valueType, + formatted, + ) return seriesList @@ -2009,7 +2146,7 @@ def alpha(requestContext, seriesList, alpha): value between 0 and 1. """ for series in seriesList: - series.options['alpha'] = alpha + series.options["alpha"] = alpha return seriesList @@ -2047,18 +2184,18 @@ def substr(requestContext, seriesList, start=0, stop=0): """ for series in seriesList: - left = series.name.rfind('(') + 1 - right = series.name.find(')') + left = series.name.rfind("(") + 1 + right = series.name.find(")") if right < 0: - right = len(series.name)+1 - cleanName = series.name[left:right:].split('.') + right = len(series.name) + 1 + cleanName = series.name[left:right:].split(".") if int(stop) == 0: - series.name = '.'.join(cleanName[int(start)::]) + series.name = ".".join(cleanName[int(start) : :]) else: - series.name = '.'.join(cleanName[int(start):int(stop):]) + series.name = ".".join(cleanName[int(start) : int(stop) :]) # substr(func(a.b,'c'),1) becomes b instead of b,'c' - series.name = re.sub(',.*$', '', series.name) + series.name = re.sub(",.*$", "", series.name) return seriesList @@ -2083,8 +2220,9 @@ def logarithm(requestContext, seriesList, base=10): else: newValues.append(math.log(val, base)) newName = "log(%s, %s)" % (series.name, base) - newSeries = TimeSeries(newName, series.start, series.end, series.step, - newValues) + newSeries = TimeSeries( + newName, series.start, series.end, series.step, newValues + ) newSeries.pathExpression = newName results.append(newSeries) return results @@ -2346,7 +2484,7 @@ def _getPercentile(points, n, interpolate=False): sortedPoints = sorted(not_none(points)) if len(sortedPoints) == 0: return None - fractionalRank = (n/100.0) * (len(sortedPoints) + 1) + fractionalRank = (n / 100.0) * (len(sortedPoints) + 1) rank = int(fractionalRank) rankFraction = fractionalRank - rank @@ -2370,23 +2508,29 @@ def _getPercentile(points, n, interpolate=False): def nPercentile(requestContext, seriesList, n): """Returns n-percent of each series in the seriesList.""" - assert n, 'The requested percent is required to be greater than 0' + assert n, "The requested percent is required to be greater than 0" results = [] for s in seriesList: # Create a sorted copy of the TimeSeries excluding None values in the # values list. - s_copy = TimeSeries(s.name, s.start, s.end, s.step, - sorted(not_none(s))) + s_copy = TimeSeries( + s.name, s.start, s.end, s.step, sorted(not_none(s)) + ) if not s_copy: - continue # Skip this series because it is empty. + continue # Skip this series because it is empty. perc_val = _getPercentile(s_copy, n) if perc_val is not None: - name = 'nPercentile(%s, %g)' % (s_copy.name, n) - point_count = int((s.end - s.start)/s.step) - perc_series = TimeSeries(name, s_copy.start, s_copy.end, - s_copy.step, [perc_val] * point_count) + name = "nPercentile(%s, %g)" % (s_copy.name, n) + point_count = int((s.end - s.start) / s.step) + perc_series = TimeSeries( + name, + s_copy.start, + s_copy.end, + s_copy.step, + [perc_val] * point_count, + ) perc_series.pathExpression = name results.append(perc_series) return results @@ -2404,8 +2548,11 @@ def averageOutsidePercentile(requestContext, seriesList, n): lowPercentile = _getPercentile(averages, 100 - n) highPercentile = _getPercentile(averages, n) - return [s for s in seriesList - if not lowPercentile < safeAvg(s) < highPercentile] + return [ + s + for s in seriesList + if not lowPercentile < safeAvg(s) < highPercentile + ] def removeBetweenPercentile(requestContext, seriesList, n): @@ -2418,12 +2565,20 @@ def removeBetweenPercentile(requestContext, seriesList, n): transposed = list(zip_longest(*seriesList)) - lowPercentiles = [_getPercentile(col, 100-n) for col in transposed] + lowPercentiles = [_getPercentile(col, 100 - n) for col in transposed] highPercentiles = [_getPercentile(col, n) for col in transposed] - return [l for l in seriesList - if sum([not lowPercentiles[index] < val < highPercentiles[index] - for index, val in enumerate(l)]) > 0] + return [ + l + for l in seriesList + if sum( + [ + not lowPercentiles[index] < val < highPercentiles[index] + for index, val in enumerate(l) + ] + ) + > 0 + ] def removeAbovePercentile(requestContext, seriesList, n): @@ -2432,7 +2587,7 @@ def removeAbovePercentile(requestContext, seriesList, n): provided. Values above this percentile are assigned a value of None. """ for s in seriesList: - s.name = 'removeAbovePercentile(%s, %g)' % (s.name, n) + s.name = "removeAbovePercentile(%s, %g)" % (s.name, n) s.pathExpression = s.name try: percentile = nPercentile(requestContext, [s], n)[0][0] @@ -2453,7 +2608,7 @@ def removeAboveValue(requestContext, seriesList, n): provided. Values above this threshold are assigned a value of None. """ for s in seriesList: - s.name = 'removeAboveValue(%s, %g)' % (s.name, n) + s.name = "removeAboveValue(%s, %g)" % (s.name, n) s.pathExpression = s.name for (index, val) in enumerate(s): if val is None: @@ -2470,7 +2625,7 @@ def removeBelowPercentile(requestContext, seriesList, n): provided. Values below this percentile are assigned a value of None. """ for s in seriesList: - s.name = 'removeBelowPercentile(%s, %g)' % (s.name, n) + s.name = "removeBelowPercentile(%s, %g)" % (s.name, n) s.pathExpression = s.name try: percentile = nPercentile(requestContext, [s], n)[0][0] @@ -2491,7 +2646,7 @@ def removeBelowValue(requestContext, seriesList, n): provided. Values below this threshold are assigned a value of None. """ for s in seriesList: - s.name = 'removeBelowValue(%s, %g)' % (s.name, n) + s.name = "removeBelowValue(%s, %g)" % (s.name, n) s.pathExpression = s.name for index, val in enumerate(s): if val is None: @@ -2645,15 +2800,19 @@ def mostDeviant(requestContext, seriesList, n): mean = safeAvg(series) if mean is None: continue - square_sum = sum([(value - mean) ** 2 for value in series - if value is not None]) + square_sum = sum( + [(value - mean) ** 2 for value in series if value is not None] + ) sigma = safeDiv(square_sum, safeLen(series)) if sigma is None: continue deviants.append((sigma, series)) - return [series for sig, series in sorted(deviants, # sort by sigma - key=itemgetter(0), - reverse=True)][:n] + return [ + series + for sig, series in sorted( + deviants, key=itemgetter(0), reverse=True # sort by sigma + ) + ][:n] def stdev(requestContext, seriesList, points, windowTolerance=0.1): @@ -2676,10 +2835,17 @@ def stdev(requestContext, seriesList, points, windowTolerance=0.1): # For this we take the standard deviation in terms of the moving average # and the moving average of series squares. for seriesIndex, series in enumerate(seriesList): - stdevSeries = TimeSeries("stdev(%s,%d)" % (series.name, int(points)), - series.start, series.end, series.step, []) - stdevSeries.pathExpression = "stdev(%s,%d)" % (series.name, - int(points)) + stdevSeries = TimeSeries( + "stdev(%s,%d)" % (series.name, int(points)), + series.start, + series.end, + series.step, + [], + ) + stdevSeries.pathExpression = "stdev(%s,%d)" % ( + series.name, + int(points), + ) validPoints = 0 currentSum = 0 @@ -2703,20 +2869,25 @@ def stdev(requestContext, seriesList, points, windowTolerance=0.1): # Remove the value that just dropped out of the window if not bootstrapping and droppedValue is not None: currentSum -= droppedValue - currentSumOfSquares -= droppedValue**2 + currentSumOfSquares -= droppedValue ** 2 # Add in the value that just popped in the window if newValue is not None: currentSum += newValue - currentSumOfSquares += newValue**2 + currentSumOfSquares += newValue ** 2 if ( - validPoints > 0 and - float(validPoints) / points >= windowTolerance + validPoints > 0 + and float(validPoints) / points >= windowTolerance ): try: - deviation = math.sqrt(validPoints * currentSumOfSquares - - currentSum**2) / validPoints + deviation = ( + math.sqrt( + validPoints * currentSumOfSquares + - currentSum ** 2 + ) + / validPoints + ) except ValueError: deviation = None stdevSeries.append(deviation) @@ -2733,15 +2904,17 @@ def secondYAxis(requestContext, seriesList): Graph the series on the secondary Y axis. """ for series in seriesList: - series.options['secondYAxis'] = True - series.name = 'secondYAxis(%s)' % series.name + series.options["secondYAxis"] = True + series.name = "secondYAxis(%s)" % series.name return seriesList -def holtWintersIntercept(alpha, actual, last_season, last_intercept, - last_slope): - return (alpha * (actual - last_season) + - (1 - alpha) * (last_intercept + last_slope)) +def holtWintersIntercept( + alpha, actual, last_season, last_intercept, last_slope +): + return alpha * (actual - last_season) + (1 - alpha) * ( + last_intercept + last_slope + ) def holtWintersSlope(beta, intercept, last_intercept, last_slope): @@ -2755,8 +2928,10 @@ def holtWintersSeasonal(gamma, actual, intercept, last_season): def holtWintersDeviation(gamma, actual, prediction, last_seasonal_dev): if prediction is None: prediction = 0 - return (gamma * math.fabs(actual - prediction) + - (1 - gamma) * last_seasonal_dev) + return ( + gamma * math.fabs(actual - prediction) + + (1 - gamma) * last_seasonal_dev + ) def holtWintersAnalysis(series): @@ -2814,16 +2989,20 @@ def getLastDeviation(i): prediction = next_pred last_seasonal = getLastSeasonal(i) - next_last_seasonal = getLastSeasonal(i+1) + next_last_seasonal = getLastSeasonal(i + 1) last_seasonal_dev = getLastDeviation(i) - intercept = holtWintersIntercept(alpha, actual, last_seasonal, - last_intercept, last_slope) + intercept = holtWintersIntercept( + alpha, actual, last_seasonal, last_intercept, last_slope + ) slope = holtWintersSlope(beta, intercept, last_intercept, last_slope) - seasonal = holtWintersSeasonal(gamma, actual, intercept, last_seasonal) + seasonal = holtWintersSeasonal( + gamma, actual, intercept, last_seasonal + ) next_pred = intercept + slope + next_last_seasonal - deviation = holtWintersDeviation(gamma, actual, prediction, - last_seasonal_dev) + deviation = holtWintersDeviation( + gamma, actual, prediction, last_seasonal_dev + ) intercepts.append(intercept) slopes.append(slope) @@ -2833,21 +3012,25 @@ def getLastDeviation(i): # make the new forecast series forecastName = "holtWintersForecast(%s)" % series.name - forecastSeries = TimeSeries(forecastName, series.start, series.end, - series.step, predictions) + forecastSeries = TimeSeries( + forecastName, series.start, series.end, series.step, predictions + ) forecastSeries.pathExpression = forecastName # make the new deviation series deviationName = "holtWintersDeviation(%s)" % series.name - deviationSeries = TimeSeries(deviationName, series.start, series.end, - series.step, deviations) + deviationSeries = TimeSeries( + deviationName, series.start, series.end, series.step, deviations + ) deviationSeries.pathExpression = deviationName - results = {'predictions': forecastSeries, - 'deviations': deviationSeries, - 'intercepts': intercepts, - 'slopes': slopes, - 'seasonals': seasonals} + results = { + "predictions": forecastSeries, + "deviations": deviationSeries, + "intercepts": intercepts, + "slopes": slopes, + "seasonals": seasonals, + } return results @@ -2859,18 +3042,22 @@ def holtWintersForecast(requestContext, seriesList): previewSeconds = 7 * 86400 # 7 days # ignore original data and pull new, including our preview newContext = requestContext.copy() - newContext['startTime'] = (requestContext['startTime'] - - timedelta(seconds=previewSeconds)) - previewList = evaluateTokens(newContext, requestContext['args'][0]) + newContext["startTime"] = requestContext["startTime"] - timedelta( + seconds=previewSeconds + ) + previewList = evaluateTokens(newContext, requestContext["args"][0]) results = [] for series in previewList: analysis = holtWintersAnalysis(series) - predictions = analysis['predictions'] + predictions = analysis["predictions"] windowPoints = previewSeconds // predictions.step - result = TimeSeries("holtWintersForecast(%s)" % series.name, - predictions.start + previewSeconds, - predictions.end, predictions.step, - predictions[windowPoints:]) + result = TimeSeries( + "holtWintersForecast(%s)" % series.name, + predictions.start + previewSeconds, + predictions.end, + predictions.step, + predictions[windowPoints:], + ) result.pathExpression = result.name results.append(result) return results @@ -2884,23 +3071,34 @@ def holtWintersConfidenceBands(requestContext, seriesList, delta=3): previewSeconds = 7 * 86400 # 7 days # ignore original data and pull new, including our preview newContext = requestContext.copy() - newContext['startTime'] = (requestContext['startTime'] - - timedelta(seconds=previewSeconds)) - previewList = evaluateTokens(newContext, requestContext['args'][0]) + newContext["startTime"] = requestContext["startTime"] - timedelta( + seconds=previewSeconds + ) + previewList = evaluateTokens(newContext, requestContext["args"][0]) results = [] for series in previewList: analysis = holtWintersAnalysis(series) - data = analysis['predictions'] + data = analysis["predictions"] windowPoints = previewSeconds // data.step - forecast = TimeSeries(data.name, data.start + previewSeconds, - data.end, data.step, data[windowPoints:]) + forecast = TimeSeries( + data.name, + data.start + previewSeconds, + data.end, + data.step, + data[windowPoints:], + ) forecast.pathExpression = data.pathExpression - data = analysis['deviations'] + data = analysis["deviations"] windowPoints = previewSeconds // data.step - deviation = TimeSeries(data.name, data.start + previewSeconds, - data.end, data.step, data[windowPoints:]) + deviation = TimeSeries( + data.name, + data.start + previewSeconds, + data.end, + data.step, + data[windowPoints:], + ) deviation.pathExpression = data.pathExpression seriesLength = len(forecast) @@ -2921,10 +3119,12 @@ def holtWintersConfidenceBands(requestContext, seriesList, delta=3): upperName = "holtWintersConfidenceUpper(%s)" % series.name lowerName = "holtWintersConfidenceLower(%s)" % series.name - upperSeries = TimeSeries(upperName, forecast.start, forecast.end, - forecast.step, upperBand) - lowerSeries = TimeSeries(lowerName, forecast.start, forecast.end, - forecast.step, lowerBand) + upperSeries = TimeSeries( + upperName, forecast.start, forecast.end, forecast.step, upperBand + ) + lowerSeries = TimeSeries( + lowerName, forecast.start, forecast.end, forecast.step, lowerBand + ) upperSeries.pathExpression = series.pathExpression lowerSeries.pathExpression = series.pathExpression results.append(lowerSeries) @@ -2939,8 +3139,9 @@ def holtWintersAberration(requestContext, seriesList, delta=3): """ results = [] for series in seriesList: - confidenceBands = holtWintersConfidenceBands(requestContext, [series], - delta) + confidenceBands = holtWintersConfidenceBands( + requestContext, [series], delta + ) lowerBand = confidenceBands[0] upperBand = confidenceBands[1] aberration = list() @@ -2955,8 +3156,11 @@ def holtWintersAberration(requestContext, seriesList, delta=3): aberration.append(0) newName = "holtWintersAberration(%s)" % series.name - results.append(TimeSeries(newName, series.start, series.end, - series.step, aberration)) + results.append( + TimeSeries( + newName, series.start, series.end, series.step, aberration + ) + ) return results @@ -2969,8 +3173,9 @@ def holtWintersConfidenceArea(requestContext, seriesList, delta=3): bands = holtWintersConfidenceBands(requestContext, seriesList, delta) results = areaBetween(requestContext, bands) for series in results: - series.name = series.name.replace('areaBetween', - 'holtWintersConfidenceArea') + series.name = series.name.replace( + "areaBetween", "holtWintersConfidenceArea" + ) return results @@ -2995,8 +3200,9 @@ def linearRegressionAnalysis(series): return factor, offset -def linearRegression(requestContext, seriesList, startSourceAt=None, - endSourceAt=None): +def linearRegression( + requestContext, seriesList, startSourceAt=None, endSourceAt=None +): """ Graphs the liner regression function by least squares method. @@ -3014,12 +3220,13 @@ def linearRegression(requestContext, seriesList, startSourceAt=None, """ from .app import evaluateTarget + results = [] sourceContext = requestContext.copy() if startSourceAt is not None: - sourceContext['startTime'] = parseATTime(startSourceAt) + sourceContext["startTime"] = parseATTime(startSourceAt) if endSourceAt is not None: - sourceContext['endTime'] = parseATTime(endSourceAt) + sourceContext["endTime"] = parseATTime(endSourceAt) sourceList = [] for series in seriesList: @@ -3027,18 +3234,22 @@ def linearRegression(requestContext, seriesList, startSourceAt=None, sourceList.extend(source) for source, series in zip(sourceList, seriesList): - newName = 'linearRegression(%s, %s, %s)' % ( + newName = "linearRegression(%s, %s, %s)" % ( series.name, - int(epoch(sourceContext['startTime'])), - int(epoch(sourceContext['endTime']))) + int(epoch(sourceContext["startTime"])), + int(epoch(sourceContext["endTime"])), + ) forecast = linearRegressionAnalysis(source) if forecast is None: continue factor, offset = forecast - values = [offset + (series.start + i * series.step) * factor - for i in range(len(series))] - newSeries = TimeSeries(newName, series.start, series.end, - series.step, values) + values = [ + offset + (series.start + i * series.step) * factor + for i in range(len(series)) + ] + newSeries = TimeSeries( + newName, series.start, series.end, series.step, values + ) newSeries.pathExpression = newSeries.name results.append(newSeries) return results @@ -3060,8 +3271,8 @@ def drawAsInfinite(requestContext, seriesList): """ for series in seriesList: - series.options['drawAsInfinite'] = True - series.name = 'drawAsInfinite(%s)' % series.name + series.options["drawAsInfinite"] = True + series.name = "drawAsInfinite(%s)" % series.name return seriesList @@ -3081,7 +3292,7 @@ def lineWidth(requestContext, seriesList, width): """ for series in seriesList: - series.options['lineWidth'] = width + series.options["lineWidth"] = width return seriesList @@ -3098,13 +3309,14 @@ def dashed(requestContext, seriesList, dashLength=5): """ for series in seriesList: - series.name = 'dashed(%s, %g)' % (series.name, dashLength) - series.options['dashed'] = dashLength + series.name = "dashed(%s, %g)" % (series.name, dashLength) + series.options["dashed"] = dashLength return seriesList -def timeStack(requestContext, seriesList, timeShiftUnit, timeShiftStart, - timeShiftEnd): +def timeStack( + requestContext, seriesList, timeShiftUnit, timeShiftStart, timeShiftEnd +): """ Takes one metric or a wildcard seriesList, followed by a quoted string with the length of time (See ``from / until`` in the render\_api_ for @@ -3125,7 +3337,7 @@ def timeStack(requestContext, seriesList, timeShiftUnit, timeShiftStart, """ # Default to negative. parseTimeOffset defaults to + if timeShiftUnit[0].isdigit(): - timeShiftUnit = '-' + timeShiftUnit + timeShiftUnit = "-" + timeShiftUnit delta = parseTimeOffset(timeShiftUnit) # if len(seriesList) > 1, they will all have the same pathExpression, # which is all we care about. @@ -3137,12 +3349,14 @@ def timeStack(requestContext, seriesList, timeShiftUnit, timeShiftStart, for shft in range(timeShiftStartint, timeShiftEndint): myContext = requestContext.copy() innerDelta = delta * shft - myContext['startTime'] = requestContext['startTime'] + innerDelta - myContext['endTime'] = requestContext['endTime'] + innerDelta + myContext["startTime"] = requestContext["startTime"] + innerDelta + myContext["endTime"] = requestContext["endTime"] + innerDelta for shiftedSeries in evaluateTarget(myContext, series.pathExpression): - shiftedSeries.name = 'timeShift(%s, %s, %s)' % (shiftedSeries.name, - timeShiftUnit, - shft) + shiftedSeries.name = "timeShift(%s, %s, %s)" % ( + shiftedSeries.name, + timeShiftUnit, + shft, + ) shiftedSeries.pathExpression = shiftedSeries.name shiftedSeries.start = series.start shiftedSeries.end = series.end @@ -3155,8 +3369,9 @@ def localDST(dt): return time.localtime(time.mktime(dt.timetuple())).tm_isdst -def timeShift(requestContext, seriesList, timeShift, resetEnd=True, - alignDST=False): +def timeShift( + requestContext, seriesList, timeShift, resetEnd=True, alignDST=False +): """ Takes one metric or a wildcard seriesList, followed by a quoted string with the length of time (See ``from / until`` in the render\_api_ for @@ -3189,37 +3404,33 @@ def timeShift(requestContext, seriesList, timeShift, resetEnd=True, """ # Default to negative. parseTimeOffset defaults to + if timeShift[0].isdigit(): - timeShift = '-' + timeShift + timeShift = "-" + timeShift delta = parseTimeOffset(timeShift) myContext = requestContext.copy() - myContext['startTime'] = requestContext['startTime'] + delta - myContext['endTime'] = requestContext['endTime'] + delta + myContext["startTime"] = requestContext["startTime"] + delta + myContext["endTime"] = requestContext["endTime"] + delta if alignDST: - reqStartDST = localDST(requestContext['startTime']) - reqEndDST = localDST(requestContext['endTime']) - myStartDST = localDST(myContext['startTime']) - myEndDST = localDST(myContext['endTime']) + reqStartDST = localDST(requestContext["startTime"]) + reqEndDST = localDST(requestContext["endTime"]) + myStartDST = localDST(myContext["startTime"]) + myEndDST = localDST(myContext["endTime"]) dstOffset = timedelta(hours=0) # If the requestContext is entirely in DST, and we are entirely # NOT in DST - if ( - (reqStartDST and reqEndDST) and - (not myStartDST and not myEndDST) - ): + if (reqStartDST and reqEndDST) and (not myStartDST and not myEndDST): dstOffset = timedelta(hours=1) # Or if the requestContext is entirely NOT in DST, and we are # entirely in DST - elif ( - (not reqStartDST and not reqEndDST) and - (myStartDST and myEndDST) + elif (not reqStartDST and not reqEndDST) and ( + myStartDST and myEndDST ): dstOffset = timedelta(hours=-1) # Otherwise, we don't do anything, because it would be visually # confusing - myContext['startTime'] += dstOffset - myContext['endTime'] += dstOffset + myContext["startTime"] += dstOffset + myContext["endTime"] += dstOffset results = [] if not seriesList: @@ -3230,20 +3441,23 @@ def timeShift(requestContext, seriesList, timeShift, resetEnd=True, series = seriesList[0] for shiftedSeries in evaluateTarget(myContext, series.pathExpression): - shiftedSeries.name = 'timeShift(%s, %s)' % (shiftedSeries.name, - timeShift) + shiftedSeries.name = "timeShift(%s, %s)" % ( + shiftedSeries.name, + timeShift, + ) if resetEnd: shiftedSeries.end = series.end else: shiftedSeries.end = ( - shiftedSeries.end - shiftedSeries.start + series.start) + shiftedSeries.end - shiftedSeries.start + series.start + ) shiftedSeries.start = series.start results.append(shiftedSeries) return results -def timeSlice(requestContext, seriesList, startSliceAt, endSliceAt='now'): +def timeSlice(requestContext, seriesList, startSliceAt, endSliceAt="now"): """ Takes one metric or a wildcard metric, followed by a quoted string with the time to start the line and another quoted string @@ -3264,8 +3478,11 @@ def timeSlice(requestContext, seriesList, startSliceAt, endSliceAt='now'): end = epoch(parseATTime(endSliceAt)) for slicedSeries in seriesList: - slicedSeries.name = 'timeSlice(%s, %s, %s)' % (slicedSeries.name, - int(start), int(end)) + slicedSeries.name = "timeSlice(%s, %s, %s)" % ( + slicedSeries.name, + int(start), + int(end), + ) curr = epoch(requestContext["startTime"]) for i, v in enumerate(slicedSeries): if v is None or curr < start or curr > end: @@ -3287,15 +3504,15 @@ def constantLine(requestContext, value): """ name = "constantLine(%s)" % str(value) - start = int(epoch(requestContext['startTime'])) - end = int(epoch(requestContext['endTime'])) + start = int(epoch(requestContext["startTime"])) + end = int(epoch(requestContext["endTime"])) step = int((end - start) / 2.0) series = TimeSeries(str(value), start, end, step, [value, value, value]) series.pathExpression = name return [series] -def aggregateLine(requestContext, seriesList, func='avg'): +def aggregateLine(requestContext, seriesList, func="avg"): """ Takes a metric or wildcard seriesList and draws a horizontal line based on the function applied to each series. @@ -3315,7 +3532,7 @@ def aggregateLine(requestContext, seriesList, func='avg'): &target=aggregateLine(server*.connections.total, 'avg') """ - t_funcs = {'avg': safeAvg, 'min': safeMin, 'max': safeMax} + t_funcs = {"avg": safeAvg, "min": safeMin, "max": safeMax} if func not in t_funcs: raise ValueError("Invalid function %s" % func) @@ -3324,9 +3541,9 @@ def aggregateLine(requestContext, seriesList, func='avg'): for series in seriesList: value = t_funcs[func](series) if value is not None: - name = 'aggregateLine(%s, %g)' % (series.name, value) + name = "aggregateLine(%s, %g)" % (series.name, value) else: - name = 'aggregateLine(%s, None)' % (series.name) + name = "aggregateLine(%s, None)" % (series.name) [series] = constantLine(requestContext, value) series.name = name @@ -3355,19 +3572,22 @@ def verticalLine(requestContext, ts, label=None, color=None): &target=verticalLine("-5mins") """ - ts = int(epoch(parseATTime(ts, requestContext['tzinfo']))) - start = int(epoch(requestContext['startTime'])) - end = int(epoch(requestContext['endTime'])) + ts = int(epoch(parseATTime(ts, requestContext["tzinfo"]))) + start = int(epoch(requestContext["startTime"])) + end = int(epoch(requestContext["endTime"])) if ts < start: - raise ValueError("verticalLine(): timestamp %s exists " - "before start of range" % ts) + raise ValueError( + "verticalLine(): timestamp %s exists " + "before start of range" % ts + ) elif ts > end: - raise ValueError("verticalLine(): timestamp %s exists " - "after end of range" % ts) + raise ValueError( + "verticalLine(): timestamp %s exists " "after end of range" % ts + ) start = end = ts step = 1.0 series = TimeSeries(label, start, end, step, [1.0, 1.0]) - series.options['drawAsInfinite'] = True + series.options["drawAsInfinite"] = True if color: series.color = color return [series] @@ -3393,7 +3613,9 @@ def threshold(requestContext, value, label=None, color=None): return [series] -def transformNull(requestContext, seriesList, default=0, referenceSeries=None): +def transformNull( + requestContext, seriesList, default=0, referenceSeries=None +): """ Takes a metric or wildcard seriesList and replaces null values with the value specified by `default`. The value 0 used if not specified. @@ -3411,6 +3633,7 @@ def transformNull(requestContext, seriesList, default=0, referenceSeries=None): This would take any page that didn't have values and supply negative 1 as a default. Any other numeric value may be used as well. """ + def transform(v, d): if v is None: return d @@ -3418,25 +3641,30 @@ def transform(v, d): return v if referenceSeries: - defaults = [default if any(v is not None for v in x) else None - for x in zip_longest(*referenceSeries)] + defaults = [ + default if any(v is not None for v in x) else None + for x in zip_longest(*referenceSeries) + ] else: defaults = None for series in seriesList: if referenceSeries: series.name = "transformNull(%s,%g,referenceSeries)" % ( - series.name, default) + series.name, + default, + ) else: series.name = "transformNull(%s,%g)" % (series.name, default) series.pathExpression = series.name if defaults: - values = [transform(v, d) for v, d in zip_longest(series, - defaults)] + values = [ + transform(v, d) for v, d in zip_longest(series, defaults) + ] else: values = [transform(v, default) for v in series] series.extend(values) - del series[:len(values)] + del series[: len(values)] return seriesList @@ -3466,7 +3694,7 @@ def transform(v): series.pathExpression = series.name values = [transform(v) for v in series] series.extend(values) - del series[:len(values)] + del series[: len(values)] return seriesList @@ -3562,8 +3790,9 @@ def mapSeries(requestContext, seriesList, mapNode): return [metaSeries[k] for k in keys] -def reduceSeries(requestContext, seriesLists, reduceFunction, reduceNode, - *reduceMatchers): +def reduceSeries( + requestContext, seriesLists, reduceFunction, reduceNode, *reduceMatchers +): """ Short form: ``reduce()``. @@ -3621,25 +3850,30 @@ def reduceSeries(requestContext, seriesLists, reduceFunction, reduceNode, keys = [] for seriesList in seriesLists: for series in seriesList: - nodes = series.name.split('.') + nodes = series.name.split(".") node = nodes[reduceNode] - reduceSeriesName = '.'.join( - nodes[0:reduceNode]) + '.reduce.' + reduceFunction + reduceSeriesName = ( + ".".join(nodes[0:reduceNode]) + ".reduce." + reduceFunction + ) if node in reduceMatchers: if reduceSeriesName not in metaSeries: - metaSeries[reduceSeriesName] = [None] * len(reduceMatchers) + metaSeries[reduceSeriesName] = [None] * len( + reduceMatchers + ) keys.append(reduceSeriesName) i = reduceMatchers.index(node) metaSeries[reduceSeriesName][i] = series for key in keys: metaSeries[key] = app.functions[reduceFunction]( - requestContext, *[[s] for s in metaSeries[key]])[0] + requestContext, *[[s] for s in metaSeries[key]] + )[0] metaSeries[key].name = key return [metaSeries[key] for key in keys] -def applyByNode(requestContext, seriesList, nodeNum, templateFunction, - newName=None): +def applyByNode( + requestContext, seriesList, nodeNum, templateFunction, newName=None +): """ Takes a seriesList and applies some complicated function (described by a string), replacing templates with unique prefixes of keys from the @@ -3660,16 +3894,17 @@ def applyByNode(requestContext, seriesList, nodeNum, templateFunction, """ from .app import evaluateTarget + prefixes = set() for series in seriesList: - prefix = '.'.join(series.name.split('.')[:nodeNum + 1]) + prefix = ".".join(series.name.split(".")[: nodeNum + 1]) prefixes.add(prefix) results = [] for prefix in sorted(prefixes): - target = templateFunction.replace('%', prefix) + target = templateFunction.replace("%", prefix) for resultSeries in evaluateTarget(requestContext, target): if newName: - resultSeries.name = newName.replace('%', prefix) + resultSeries.name = newName.replace("%", prefix) resultSeries.pathExpression = prefix resultSeries.start = series.start resultSeries.end = series.end @@ -3719,20 +3954,22 @@ def groupByNodes(requestContext, seriesList, callback, *nodes): """ from .app import app + metaSeries = {} keys = [] if isinstance(nodes, int): nodes = [nodes] for series in seriesList: - key = '.'.join(series.name.split(".")[n] for n in nodes) + key = ".".join(series.name.split(".")[n] for n in nodes) if key not in metaSeries: metaSeries[key] = [series] keys.append(key) else: metaSeries[key].append(series) for key in metaSeries: - metaSeries[key] = app.functions[callback](requestContext, - metaSeries[key])[0] + metaSeries[key] = app.functions[callback]( + requestContext, metaSeries[key] + )[0] metaSeries[key].name = key return [metaSeries[key] for key in keys] @@ -3764,7 +4001,7 @@ def grep(requestContext, seriesList, pattern): return [s for s in seriesList if regex.search(s.name)] -def smartSummarize(requestContext, seriesList, intervalString, func='sum'): +def smartSummarize(requestContext, seriesList, intervalString, func="sum"): """ Smarter experimental version of summarize. """ @@ -3774,17 +4011,20 @@ def smartSummarize(requestContext, seriesList, intervalString, func='sum'): # Adjust the start time to fit an entire day for intervals >= 1 day requestContext = requestContext.copy() - tzinfo = requestContext['tzinfo'] - s = requestContext['startTime'] + tzinfo = requestContext["tzinfo"] + s = requestContext["startTime"] if interval >= DAY: - requestContext['startTime'] = datetime(s.year, s.month, s.day, - tzinfo=tzinfo) + requestContext["startTime"] = datetime( + s.year, s.month, s.day, tzinfo=tzinfo + ) elif interval >= HOUR: - requestContext['startTime'] = datetime(s.year, s.month, s.day, s.hour, - tzinfo=tzinfo) + requestContext["startTime"] = datetime( + s.year, s.month, s.day, s.hour, tzinfo=tzinfo + ) elif interval >= MINUTE: - requestContext['startTime'] = datetime(s.year, s.month, s.day, s.hour, - s.minute, tzinfo=tzinfo) + requestContext["startTime"] = datetime( + s.year, s.month, s.day, s.hour, s.minute, tzinfo=tzinfo + ) paths = [] for series in seriesList: @@ -3794,10 +4034,10 @@ def smartSummarize(requestContext, seriesList, intervalString, func='sum'): for series in seriesList: # XXX: breaks with summarize(metric.{a,b}) # each series.pathExpression == metric.{a,b} - newSeries = evaluateTarget(requestContext, - series.pathExpression, - data_store)[0] - series[0:len(series)] = newSeries + newSeries = evaluateTarget( + requestContext, series.pathExpression, data_store + )[0] + series[0 : len(series)] = newSeries series.start = newSeries.start series.end = newSeries.end series.step = newSeries.step @@ -3805,8 +4045,9 @@ def smartSummarize(requestContext, seriesList, intervalString, func='sum'): for series in seriesList: buckets = {} # {timestamp: [values]} - timestamps = range(int(series.start), int(series.end), - int(series.step)) + timestamps = range( + int(series.start), int(series.end), int(series.step) + ) datapoints = zip_longest(timestamps, series) # Populate buckets @@ -3829,33 +4070,37 @@ def smartSummarize(requestContext, seriesList, intervalString, func='sum'): bucket = buckets.get(bucketInterval, []) if bucket: - if func == 'avg': + if func == "avg": newValues.append(float(sum(bucket)) / float(len(bucket))) - elif func == 'last': - newValues.append(bucket[len(bucket)-1]) - elif func == 'max': + elif func == "last": + newValues.append(bucket[len(bucket) - 1]) + elif func == "max": newValues.append(max(bucket)) - elif func == 'min': + elif func == "min": newValues.append(min(bucket)) else: newValues.append(sum(bucket)) else: newValues.append(None) - newName = "smartSummarize(%s, \"%s\", \"%s\")" % (series.name, - intervalString, - func) + newName = 'smartSummarize(%s, "%s", "%s")' % ( + series.name, + intervalString, + func, + ) alignedEnd = series.start + (bucketInterval * interval) + interval - newSeries = TimeSeries(newName, series.start, alignedEnd, interval, - newValues) + newSeries = TimeSeries( + newName, series.start, alignedEnd, interval, newValues + ) newSeries.pathExpression = newName results.append(newSeries) return results -def summarize(requestContext, seriesList, intervalString, func='sum', - alignToFrom=False): +def summarize( + requestContext, seriesList, intervalString, func="sum", alignToFrom=False +): """ Summarize the data into interval buckets of a certain size. @@ -3902,8 +4147,9 @@ def summarize(requestContext, seriesList, intervalString, func='sum', for series in seriesList: buckets = {} - timestamps = range(int(series.start), int(series.end) + 1, - int(series.step)) + timestamps = range( + int(series.start), int(series.end) + 1, int(series.step) + ) datapoints = zip_longest(timestamps, series) for timestamp, value in datapoints: @@ -3938,13 +4184,13 @@ def summarize(requestContext, seriesList, intervalString, func='sum', bucket = buckets.get(bucketInterval, []) if bucket: - if func == 'avg': + if func == "avg": newValues.append(float(sum(bucket)) / float(len(bucket))) - elif func == 'last': - newValues.append(bucket[len(bucket)-1]) - elif func == 'max': + elif func == "last": + newValues.append(bucket[len(bucket) - 1]) + elif func == "max": newValues.append(max(bucket)) - elif func == 'min': + elif func == "min": newValues.append(min(bucket)) else: newValues.append(sum(bucket)) @@ -3954,8 +4200,12 @@ def summarize(requestContext, seriesList, intervalString, func='sum', if alignToFrom: newEnd += interval - newName = "summarize(%s, \"%s\", \"%s\"%s)" % ( - series.name, intervalString, func, alignToFrom and ", true" or "") + newName = 'summarize(%s, "%s", "%s"%s)' % ( + series.name, + intervalString, + func, + alignToFrom and ", true" or "", + ) newSeries = TimeSeries(newName, newStart, newEnd, interval, newValues) newSeries.pathExpression = newName results.append(newSeries) @@ -3963,8 +4213,9 @@ def summarize(requestContext, seriesList, intervalString, func='sum', return results -def hitcount(requestContext, seriesList, intervalString, - alignToInterval=False): +def hitcount( + requestContext, seriesList, intervalString, alignToInterval=False +): """ Estimate hit counts from a list of time series. @@ -3982,41 +4233,46 @@ def hitcount(requestContext, seriesList, intervalString, if alignToInterval: requestContext = requestContext.copy() - tzinfo = requestContext['tzinfo'] - s = requestContext['startTime'] + tzinfo = requestContext["tzinfo"] + s = requestContext["startTime"] if interval >= DAY: - requestContext['startTime'] = datetime(s.year, s.month, s.day, - tzinfo=tzinfo) + requestContext["startTime"] = datetime( + s.year, s.month, s.day, tzinfo=tzinfo + ) elif interval >= HOUR: - requestContext['startTime'] = datetime(s.year, s.month, s.day, - s.hour, tzinfo=tzinfo) + requestContext["startTime"] = datetime( + s.year, s.month, s.day, s.hour, tzinfo=tzinfo + ) elif interval >= MINUTE: - requestContext['startTime'] = datetime(s.year, s.month, s.day, - s.hour, s.minute, - tzinfo=tzinfo) + requestContext["startTime"] = datetime( + s.year, s.month, s.day, s.hour, s.minute, tzinfo=tzinfo + ) # Gather all paths first, then the data paths = [] for series in seriesList: - paths.extend(pathsFromTarget(requestContext, - series.pathExpression)) + paths.extend( + pathsFromTarget(requestContext, series.pathExpression) + ) data_store = fetchData(requestContext, paths) for series in seriesList: - newSeries = evaluateTarget(requestContext, - series.pathExpression, - data_store)[0] + newSeries = evaluateTarget( + requestContext, series.pathExpression, data_store + )[0] intervalCount = int((series.end - series.start) / interval) - series[0:len(series)] = newSeries + series[0 : len(series)] = newSeries series.start = newSeries.start - series.end = newSeries.start + ( - intervalCount * interval) + interval + series.end = ( + newSeries.start + (intervalCount * interval) + interval + ) series.step = newSeries.step for series in seriesList: step = int(series.step) - bucket_count = int(math.ceil( - float(series.end - series.start) / interval)) + bucket_count = int( + math.ceil(float(series.end - series.start) / interval) + ) buckets = [[] for _ in range(bucket_count)] newStart = int(series.end - bucket_count * interval) @@ -4036,13 +4292,16 @@ def hitcount(requestContext, seriesList, intervalString, if start_bucket == end_bucket: # All of the hits go to a single bucket. if start_bucket >= 0: - buckets[start_bucket].append(value * (end_mod - start_mod)) + buckets[start_bucket].append( + value * (end_mod - start_mod) + ) else: # Spread the hits among 2 or more buckets. if start_bucket >= 0: buckets[start_bucket].append( - value * (interval - start_mod)) + value * (interval - start_mod) + ) hits_per_bucket = value * interval for j in range(start_bucket + 1, end_bucket): buckets[j].append(hits_per_bucket) @@ -4056,10 +4315,14 @@ def hitcount(requestContext, seriesList, intervalString, else: newValues.append(None) - newName = 'hitcount(%s, "%s"%s)' % (series.name, intervalString, - alignToInterval and ", true" or "") - newSeries = TimeSeries(newName, newStart, series.end, interval, - newValues) + newName = 'hitcount(%s, "%s"%s)' % ( + series.name, + intervalString, + alignToInterval and ", true" or "", + ) + newSeries = TimeSeries( + newName, newStart, series.end, interval, newValues + ) newSeries.pathExpression = newName results.append(newSeries) @@ -4118,14 +4381,17 @@ def sinFunction(requestContext, name, amplitude=1, step=60): values = [] while when < requestContext["endTime"]: - values.append(math.sin(epoch(when))*amplitude) + values.append(math.sin(epoch(when)) * amplitude) when += delta series = TimeSeries( - name, int(epoch(requestContext["startTime"])), + name, + int(epoch(requestContext["startTime"])), int(epoch(requestContext["endTime"])), - step, values) - series.pathExpression = 'sin({0})'.format(name) + step, + values, + ) + series.pathExpression = "sin({0})".format(name) return [series] @@ -4169,10 +4435,15 @@ def randomWalkFunction(requestContext, name, step=60): current += random.random() - 0.5 when += delta - return [TimeSeries( - name, int(epoch(requestContext["startTime"])), - int(epoch(requestContext["endTime"])), - step, values)] + return [ + TimeSeries( + name, + int(epoch(requestContext["startTime"])), + int(epoch(requestContext["endTime"])), + step, + values, + ) + ] def pieAverage(requestContext, series): @@ -4188,149 +4459,143 @@ def pieMinimum(requestContext, series): PieFunctions = { - 'average': pieAverage, - 'maximum': pieMaximum, - 'minimum': pieMinimum, + "average": pieAverage, + "maximum": pieMaximum, + "minimum": pieMinimum, } SeriesFunctions = { # Combine functions - 'sumSeries': sumSeries, - 'sum': sumSeries, - 'multiplySeries': multiplySeries, - 'averageSeries': averageSeries, - 'stddevSeries': stddevSeries, - 'avg': averageSeries, - 'sumSeriesWithWildcards': sumSeriesWithWildcards, - 'averageSeriesWithWildcards': averageSeriesWithWildcards, - 'multiplySeriesWithWildcards': multiplySeriesWithWildcards, - 'minSeries': minSeries, - 'maxSeries': maxSeries, - 'rangeOfSeries': rangeOfSeries, - 'percentileOfSeries': percentileOfSeries, - 'countSeries': countSeries, - 'weightedAverage': weightedAverage, - + "sumSeries": sumSeries, + "sum": sumSeries, + "multiplySeries": multiplySeries, + "averageSeries": averageSeries, + "stddevSeries": stddevSeries, + "avg": averageSeries, + "sumSeriesWithWildcards": sumSeriesWithWildcards, + "averageSeriesWithWildcards": averageSeriesWithWildcards, + "multiplySeriesWithWildcards": multiplySeriesWithWildcards, + "minSeries": minSeries, + "maxSeries": maxSeries, + "rangeOfSeries": rangeOfSeries, + "percentileOfSeries": percentileOfSeries, + "countSeries": countSeries, + "weightedAverage": weightedAverage, # Transform functions - 'scale': scale, - 'scaleToSeconds': scaleToSeconds, - 'offset': offset, - 'offsetToZero': offsetToZero, - 'derivative': derivative, - 'delay': delay, - 'squareRoot': squareRoot, - 'pow': pow, - 'powSeries': powSeries, - 'perSecond': perSecond, - 'integral': integral, - 'integralByInterval': integralByInterval, - 'nonNegativeDerivative': nonNegativeDerivative, - 'log': logarithm, - 'invert': invert, - 'timeStack': timeStack, - 'timeShift': timeShift, - 'timeSlice': timeSlice, - 'summarize': summarize, - 'smartSummarize': smartSummarize, - 'hitcount': hitcount, - 'absolute': absolute, - 'interpolate': interpolate, - + "scale": scale, + "scaleToSeconds": scaleToSeconds, + "offset": offset, + "offsetToZero": offsetToZero, + "derivative": derivative, + "delay": delay, + "squareRoot": squareRoot, + "pow": pow, + "powSeries": powSeries, + "perSecond": perSecond, + "integral": integral, + "integralByInterval": integralByInterval, + "nonNegativeDerivative": nonNegativeDerivative, + "log": logarithm, + "invert": invert, + "timeStack": timeStack, + "timeShift": timeShift, + "timeSlice": timeSlice, + "summarize": summarize, + "smartSummarize": smartSummarize, + "hitcount": hitcount, + "absolute": absolute, + "interpolate": interpolate, # Calculate functions - 'movingAverage': movingAverage, - 'movingMedian': movingMedian, - 'movingSum': movingSum, - 'movingMin': movingMin, - 'movingMax': movingMax, - 'stdev': stdev, - 'holtWintersForecast': holtWintersForecast, - 'holtWintersConfidenceBands': holtWintersConfidenceBands, - 'holtWintersConfidenceArea': holtWintersConfidenceArea, - 'holtWintersAberration': holtWintersAberration, - 'linearRegression': linearRegression, - 'asPercent': asPercent, - 'pct': asPercent, - 'diffSeries': diffSeries, - 'divideSeriesLists': divideSeriesLists, - 'divideSeries': divideSeries, - 'exponentialMovingAverage': exponentialMovingAverage, - + "movingAverage": movingAverage, + "movingMedian": movingMedian, + "movingSum": movingSum, + "movingMin": movingMin, + "movingMax": movingMax, + "stdev": stdev, + "holtWintersForecast": holtWintersForecast, + "holtWintersConfidenceBands": holtWintersConfidenceBands, + "holtWintersConfidenceArea": holtWintersConfidenceArea, + "holtWintersAberration": holtWintersAberration, + "linearRegression": linearRegression, + "asPercent": asPercent, + "pct": asPercent, + "diffSeries": diffSeries, + "divideSeriesLists": divideSeriesLists, + "divideSeries": divideSeries, + "exponentialMovingAverage": exponentialMovingAverage, # Series Filter functions - 'fallbackSeries': fallbackSeries, - 'mostDeviant': mostDeviant, - 'highestCurrent': highestCurrent, - 'lowestCurrent': lowestCurrent, - 'highestMax': highestMax, - 'currentAbove': currentAbove, - 'currentBelow': currentBelow, - 'highestAverage': highestAverage, - 'lowestAverage': lowestAverage, - 'averageAbove': averageAbove, - 'averageBelow': averageBelow, - 'maximumAbove': maximumAbove, - 'minimumAbove': minimumAbove, - 'maximumBelow': maximumBelow, - 'minimumBelow': minimumBelow, - 'nPercentile': nPercentile, - 'limit': limit, - 'sortByTotal': sortByTotal, - 'sortByName': sortByName, - 'averageOutsidePercentile': averageOutsidePercentile, - 'removeBetweenPercentile': removeBetweenPercentile, - 'sortByMaxima': sortByMaxima, - 'sortByMinima': sortByMinima, - 'useSeriesAbove': useSeriesAbove, - 'exclude': exclude, - 'grep': grep, - 'removeEmptySeries': removeEmptySeries, - + "fallbackSeries": fallbackSeries, + "mostDeviant": mostDeviant, + "highestCurrent": highestCurrent, + "lowestCurrent": lowestCurrent, + "highestMax": highestMax, + "currentAbove": currentAbove, + "currentBelow": currentBelow, + "highestAverage": highestAverage, + "lowestAverage": lowestAverage, + "averageAbove": averageAbove, + "averageBelow": averageBelow, + "maximumAbove": maximumAbove, + "minimumAbove": minimumAbove, + "maximumBelow": maximumBelow, + "minimumBelow": minimumBelow, + "nPercentile": nPercentile, + "limit": limit, + "sortByTotal": sortByTotal, + "sortByName": sortByName, + "averageOutsidePercentile": averageOutsidePercentile, + "removeBetweenPercentile": removeBetweenPercentile, + "sortByMaxima": sortByMaxima, + "sortByMinima": sortByMinima, + "useSeriesAbove": useSeriesAbove, + "exclude": exclude, + "grep": grep, + "removeEmptySeries": removeEmptySeries, # Data Filter functions - 'removeAbovePercentile': removeAbovePercentile, - 'removeAboveValue': removeAboveValue, - 'removeBelowPercentile': removeBelowPercentile, - 'removeBelowValue': removeBelowValue, - + "removeAbovePercentile": removeAbovePercentile, + "removeAboveValue": removeAboveValue, + "removeBelowPercentile": removeBelowPercentile, + "removeBelowValue": removeBelowValue, # Special functions - 'legendValue': legendValue, - 'alias': alias, - 'aliasSub': aliasSub, - 'aliasByNode': aliasByNode, - 'aliasByMetric': aliasByMetric, - 'cactiStyle': cactiStyle, - 'color': color, - 'alpha': alpha, - 'cumulative': cumulative, - 'consolidateBy': consolidateBy, - 'keepLastValue': keepLastValue, - 'changed': changed, - 'drawAsInfinite': drawAsInfinite, - 'secondYAxis': secondYAxis, - 'lineWidth': lineWidth, - 'dashed': dashed, - 'substr': substr, - 'group': group, - 'map': mapSeries, - 'mapSeries': mapSeries, - 'reduce': reduceSeries, - 'reduceSeries': reduceSeries, - 'applyByNode': applyByNode, - 'groupByNode': groupByNode, - 'groupByNodes': groupByNodes, - 'constantLine': constantLine, - 'stacked': stacked, - 'areaBetween': areaBetween, - 'verticalLine': verticalLine, - 'threshold': threshold, - 'transformNull': transformNull, - 'isNonNull': isNonNull, - 'identity': identity, - 'aggregateLine': aggregateLine, - + "legendValue": legendValue, + "alias": alias, + "aliasSub": aliasSub, + "aliasByNode": aliasByNode, + "aliasByMetric": aliasByMetric, + "cactiStyle": cactiStyle, + "color": color, + "alpha": alpha, + "cumulative": cumulative, + "consolidateBy": consolidateBy, + "keepLastValue": keepLastValue, + "changed": changed, + "drawAsInfinite": drawAsInfinite, + "secondYAxis": secondYAxis, + "lineWidth": lineWidth, + "dashed": dashed, + "substr": substr, + "group": group, + "map": mapSeries, + "mapSeries": mapSeries, + "reduce": reduceSeries, + "reduceSeries": reduceSeries, + "applyByNode": applyByNode, + "groupByNode": groupByNode, + "groupByNodes": groupByNodes, + "constantLine": constantLine, + "stacked": stacked, + "areaBetween": areaBetween, + "verticalLine": verticalLine, + "threshold": threshold, + "transformNull": transformNull, + "isNonNull": isNonNull, + "identity": identity, + "aggregateLine": aggregateLine, # test functions - 'time': timeFunction, + "time": timeFunction, "sin": sinFunction, "randomWalk": randomWalkFunction, - 'timeFunction': timeFunction, + "timeFunction": timeFunction, "sinFunction": sinFunction, "randomWalkFunction": randomWalkFunction, } diff --git a/graphite_api/intervals.py b/graphite_api/intervals.py index fdd9371..670d347 100644 --- a/graphite_api/intervals.py +++ b/graphite_api/intervals.py @@ -1,9 +1,9 @@ -INFINITY = float('inf') +INFINITY = float("inf") NEGATIVE_INFINITY = -INFINITY class IntervalSet(object): - __slots__ = ('intervals', 'size') + __slots__ = ("intervals", "size") def __init__(self, intervals, disjoint=False): self.intervals = intervals @@ -30,6 +30,7 @@ def __getitem__(self, i): def __bool__(self): return self.size != 0 + __nonzero__ = __bool__ # python 2 def __sub__(self, other): @@ -55,17 +56,22 @@ def intersect(self, other): if not self or not other: return IntervalSet([]) - intersections = [x for x in (i.intersect(j) - for i in self.intervals - for j in other.intervals) - if x] + intersections = [ + x + for x in ( + i.intersect(j) + for i in self.intervals + for j in other.intervals + ) + if x + ] return IntervalSet(intersections, disjoint=True) def intersect_interval(self, interval): - intersections = [x for x in (i.intersect(interval) - for i in self.intervals) - if x] + intersections = [ + x for x in (i.intersect(interval) for i in self.intervals) if x + ] return IntervalSet(intersections, disjoint=True) def union(self, other): @@ -73,11 +79,13 @@ def union(self, other): class Interval(object): - __slots__ = ('start', 'end', 'tuple', 'size') + __slots__ = ("start", "end", "tuple", "size") def __init__(self, start, end): if end - start < 0: - raise ValueError("Invalid interval start=%s end=%s" % (start, end)) + raise ValueError( + "Invalid interval start=%s end=%s" % (start, end) + ) self.start = start self.end = end @@ -94,15 +102,18 @@ def __lt__(self, other): return (self.start < other.start) - (self.start > other.start) def __len__(self): - raise TypeError("len() doesn't support infinite values, use the " - "'size' attribute instead") + raise TypeError( + "len() doesn't support infinite values, use the " + "'size' attribute instead" + ) def __bool__(self): return self.size != 0 + __nonzero__ = __bool__ # python 2 def __repr__(self): - return '' % str(self.tuple) + return "" % str(self.tuple) def intersect(self, other): start = max(self.start, other.start) diff --git a/graphite_api/middleware.py b/graphite_api/middleware.py index eff7868..767b454 100644 --- a/graphite_api/middleware.py +++ b/graphite_api/middleware.py @@ -5,28 +5,30 @@ class CORS(object): """ Simple middleware that adds CORS headers. """ + def __init__(self, app, origins=None): self.app = app self.origins = origins def __call__(self, environ, start_response): - origin = environ.get('HTTP_ORIGIN') + origin = environ.get("HTTP_ORIGIN") if origin is None or self.origins is None: return self.app(environ, start_response) netloc = urlparse(origin).netloc - if netloc in self.origins or '*' in self.origins: + if netloc in self.origins or "*" in self.origins: allow_origin = [ - ('Access-Control-Allow-Origin', origin), - ('Access-Control-Allow-Credentials', 'true'), + ("Access-Control-Allow-Origin", origin), + ("Access-Control-Allow-Credentials", "true"), ] - if environ['REQUEST_METHOD'] == 'OPTIONS': - start_response('204 No Content', allow_origin) + if environ["REQUEST_METHOD"] == "OPTIONS": + start_response("204 No Content", allow_origin) return [] def custom_start_response(status, headers, exc_info=None): headers.extend(allow_origin) return start_response(status, headers, exc_info) + else: custom_start_response = start_response return self.app(environ, custom_start_response) @@ -36,11 +38,12 @@ class TrailingSlash(object): """ Middleware that strips trailing slashes from URLs. """ + def __init__(self, app): self.app = app def __call__(self, environ, start_response): - path_info = environ['PATH_INFO'] - if len(path_info) > 1 and path_info.endswith('/'): - environ['PATH_INFO'] = path_info.rstrip('/') + path_info = environ["PATH_INFO"] + if len(path_info) > 1 and path_info.endswith("/"): + environ["PATH_INFO"] = path_info.rstrip("/") return self.app(environ, start_response) diff --git a/graphite_api/node.py b/graphite_api/node.py index 2891bfe..f0218a3 100644 --- a/graphite_api/node.py +++ b/graphite_api/node.py @@ -1,14 +1,14 @@ class Node(object): - __slots__ = ('name', 'path', 'local', 'is_leaf') + __slots__ = ("name", "path", "local", "is_leaf") def __init__(self, path): self.path = path - self.name = path.split('.')[-1] + self.name = path.split(".")[-1] self.local = True self.is_leaf = False def __repr__(self): - return '<%s[%x]: %s>' % (self.__class__.__name__, id(self), self.path) + return "<%s[%x]: %s>" % (self.__class__.__name__, id(self), self.path) class BranchNode(Node): @@ -16,7 +16,7 @@ class BranchNode(Node): class LeafNode(Node): - __slots__ = ('reader', 'is_leaf') + __slots__ = ("reader", "is_leaf") def __init__(self, path, reader): super(LeafNode, self).__init__(path) @@ -25,7 +25,9 @@ def __init__(self, path, reader): def fetch(self, startTime, endTime, now=None, requestContext=None): try: - result = self.reader.fetch(startTime, endTime, now, requestContext) + result = self.reader.fetch( + startTime, endTime, now, requestContext + ) except TypeError: result = self.reader.fetch(startTime, endTime) @@ -36,4 +38,4 @@ def intervals(self): return self.reader.get_intervals() def __repr__(self): - return '' % (id(self), self.path, self.reader) + return "" % (id(self), self.path, self.reader) diff --git a/graphite_api/readers.py b/graphite_api/readers.py index 4ca4124..ee2c91a 100644 --- a/graphite_api/readers.py +++ b/graphite_api/readers.py @@ -6,7 +6,7 @@ class MultiReader(object): - __slots__ = ('nodes',) + __slots__ = ("nodes",) def __init__(self, nodes): self.nodes = nodes @@ -23,8 +23,9 @@ def fetch(self, startTime, endTime, now=None, requestContext=None): for node in self.nodes: try: - results.append(node.fetch(startTime, endTime, now, - requestContext)) + results.append( + node.fetch(startTime, endTime, now, requestContext) + ) except Exception: logger.error("fetch error", exc_info=True) diff --git a/graphite_api/render/attime.py b/graphite_api/render/attime.py index 356bc35..5310123 100644 --- a/graphite_api/render/attime.py +++ b/graphite_api/render/attime.py @@ -16,79 +16,98 @@ import pytz -months = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', - 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'] -weekdays = ['sun', 'mon', 'tue', 'wed', 'thu', 'fri', 'sat'] +months = [ + "jan", + "feb", + "mar", + "apr", + "may", + "jun", + "jul", + "aug", + "sep", + "oct", + "nov", + "dec", +] +weekdays = ["sun", "mon", "tue", "wed", "thu", "fri", "sat"] def parseATTime(s, tzinfo=None, now=None): if tzinfo is None: from ..app import app - tzinfo = pytz.timezone(app.config['TIME_ZONE']) - s = s.strip().lower().replace('_', '').replace(',', '').replace(' ', '') + + tzinfo = pytz.timezone(app.config["TIME_ZONE"]) + s = s.strip().lower().replace("_", "").replace(",", "").replace(" ", "") if s.isdigit(): if ( - len(s) == 8 and - int(s[:4]) > 1900 and - int(s[4:6]) < 13 and - int(s[6:]) < 32 + len(s) == 8 + and int(s[:4]) > 1900 + and int(s[4:6]) < 13 + and int(s[6:]) < 32 ): pass # Fall back because its not a timestamp, its YYYYMMDD form else: return datetime.fromtimestamp(int(s), tzinfo) - elif ':' in s and len(s) == 13: - return tzinfo.localize(datetime.strptime(s, '%H:%M%Y%m%d'), daylight) - if '+' in s: - ref, offset = s.split('+', 1) - offset = '+' + offset - elif '-' in s: - ref, offset = s.split('-', 1) - offset = '-' + offset + elif ":" in s and len(s) == 13: + return tzinfo.localize(datetime.strptime(s, "%H:%M%Y%m%d"), daylight) + if "+" in s: + ref, offset = s.split("+", 1) + offset = "+" + offset + elif "-" in s: + ref, offset = s.split("-", 1) + offset = "-" + offset else: - ref, offset = s, '' - return (parseTimeReference(ref or now) + - parseTimeOffset(offset)).astimezone(tzinfo) + ref, offset = s, "" + return ( + parseTimeReference(ref or now) + parseTimeOffset(offset) + ).astimezone(tzinfo) def parseTimeReference(ref): if isinstance(ref, datetime): return ref - if not ref or ref == 'now': + if not ref or ref == "now": return datetime.utcnow().replace(tzinfo=pytz.utc) # Time-of-day reference - i = ref.find(':') + i = ref.find(":") hour, min = 0, 0 if i != -1: hour = int(ref[:i]) - min = int(ref[i+1:i+3]) - ref = ref[i+3:] - if ref[:2] == 'am': + min = int(ref[i + 1 : i + 3]) + ref = ref[i + 3 :] + if ref[:2] == "am": ref = ref[2:] - elif ref[:2] == 'pm': + elif ref[:2] == "pm": hour = (hour + 12) % 24 ref = ref[2:] - if ref.startswith('noon'): + if ref.startswith("noon"): hour, min = 12, 0 ref = ref[4:] - elif ref.startswith('midnight'): + elif ref.startswith("midnight"): hour, min = 0, 0 ref = ref[8:] - elif ref.startswith('teatime'): + elif ref.startswith("teatime"): hour, min = 16, 0 ref = ref[7:] - refDate = datetime.utcnow().replace(hour=hour, minute=min, second=0, - tzinfo=pytz.utc) + refDate = datetime.utcnow().replace( + hour=hour, minute=min, second=0, tzinfo=pytz.utc + ) # Day reference - if ref in ('yesterday', 'today', 'tomorrow'): # yesterday, today, tomorrow - if ref == 'yesterday': + if ref in ( + "yesterday", + "today", + "tomorrow", + ): # yesterday, today, tomorrow + if ref == "yesterday": refDate = refDate - timedelta(days=1) - if ref == 'tomorrow': + if ref == "tomorrow": refDate = refDate + timedelta(days=1) - elif ref.count('/') == 2: # MM/DD/YY[YY] - m, d, y = map(int, ref.split('/')) + elif ref.count("/") == 2: # MM/DD/YY[YY] + m, d, y = map(int, ref.split("/")) if y < 1900: y += 1900 if y < 1970: @@ -96,8 +115,9 @@ def parseTimeReference(ref): refDate = replace_date(refDate, y, m, d) elif len(ref) == 8 and ref.isdigit(): # YYYYMMDD - refDate = replace_date(refDate, int(ref[:4]), int(ref[4:6]), - int(ref[6:8])) + refDate = replace_date( + refDate, int(ref[:4]), int(ref[4:6]), int(ref[6:8]) + ) elif ref[:3] in months: # MonthName DayOfMonth month = months.index(ref[:3]) + 1 @@ -145,26 +165,26 @@ def parseTimeOffset(offset): if offset[0].isdigit(): sign = 1 else: - sign = {'+': 1, '-': -1}[offset[0]] + sign = {"+": 1, "-": -1}[offset[0]] offset = offset[1:] while offset: i = 1 while offset[:i].isdigit() and i <= len(offset): i += 1 - num = int(offset[:i-1]) - offset = offset[i-1:] + num = int(offset[: i - 1]) + offset = offset[i - 1 :] i = 1 while offset[:i].isalpha() and i <= len(offset): i += 1 - unit = offset[:i-1] - offset = offset[i-1:] + unit = offset[: i - 1] + offset = offset[i - 1 :] unitString = getUnitString(unit) - if unitString == 'months': - unitString = 'days' + if unitString == "months": + unitString = "days" num = num * 30 - if unitString == 'years': - unitString = 'days' + if unitString == "years": + unitString = "days" num = num * 365 t += timedelta(**{unitString: sign * num}) @@ -172,18 +192,18 @@ def parseTimeOffset(offset): def getUnitString(s): - if s.startswith('s'): - return 'seconds' - if s.startswith('min'): - return 'minutes' - if s.startswith('h'): - return 'hours' - if s.startswith('d'): - return 'days' - if s.startswith('w'): - return 'weeks' - if s.startswith('mon'): - return 'months' - if s.startswith('y'): - return 'years' + if s.startswith("s"): + return "seconds" + if s.startswith("min"): + return "minutes" + if s.startswith("h"): + return "hours" + if s.startswith("d"): + return "days" + if s.startswith("w"): + return "weeks" + if s.startswith("mon"): + return "months" + if s.startswith("y"): + return "years" raise Exception("Invalid offset unit '%s'" % s) diff --git a/graphite_api/render/datalib.py b/graphite_api/render/datalib.py index 9f66d6c..37b1351 100644 --- a/graphite_api/render/datalib.py +++ b/graphite_api/render/datalib.py @@ -21,7 +21,7 @@ class TimeSeries(list): - def __init__(self, name, start, end, step, values, consolidate='average'): + def __init__(self, name, start, end, step, values, consolidate="average"): list.__init__(self, values) self.name = name self.start = start @@ -35,19 +35,36 @@ def __init__(self, name, start, end, step, values, consolidate='average'): def __eq__(self, other): if isinstance(other, TimeSeries): color_eq = True - if hasattr(self, 'color'): - if hasattr(other, 'color'): - color_eq = (self.color == other.color) + if hasattr(self, "color"): + if hasattr(other, "color"): + color_eq = self.color == other.color else: color_eq = False - elif hasattr(other, 'color'): + elif hasattr(other, "color"): color_eq = False - return ((self.name, self.start, self.step, self.consolidationFunc, - self.valuesPerPoint, self.options) == - (other.name, other.start, other.step, - other.consolidationFunc, other.valuesPerPoint, - other.options)) and list.__eq__(self, other) and color_eq + return ( + ( + ( + self.name, + self.start, + self.step, + self.consolidationFunc, + self.valuesPerPoint, + self.options, + ) + == ( + other.name, + other.start, + other.step, + other.consolidationFunc, + other.valuesPerPoint, + other.options, + ) + ) + and list.__eq__(self, other) + and color_eq + ) return False def __iter__(self): @@ -83,20 +100,25 @@ def __consolidate(self, values): usable = [v for v in values if v is not None] if not usable: return None - if self.consolidationFunc == 'sum': + if self.consolidationFunc == "sum": return sum(usable) - if self.consolidationFunc == 'average': + if self.consolidationFunc == "average": return float(sum(usable)) / len(usable) - if self.consolidationFunc == 'max': + if self.consolidationFunc == "max": return max(usable) - if self.consolidationFunc == 'min': + if self.consolidationFunc == "min": return min(usable) raise Exception( - "Invalid consolidation function: '%s'" % self.consolidationFunc) + "Invalid consolidation function: '%s'" % self.consolidationFunc + ) def __repr__(self): - return 'TimeSeries(name=%s, start=%s, end=%s, step=%s)' % ( - self.name, self.start, self.end, self.step) + return "TimeSeries(name=%s, start=%s, end=%s, step=%s)" % ( + self.name, + self.start, + self.end, + self.step, + ) class DataStore(object): @@ -104,6 +126,7 @@ class DataStore(object): Simple object to store results of multi fetches. Also aids in looking up data by pathExpressions. """ + def __init__(self): self.paths = defaultdict(set) self.data = defaultdict(list) @@ -121,23 +144,20 @@ def add_data(self, path, time_info, data, exprs): # Dont add if empty if not nonempty(data): for d in self.data[path]: - if nonempty(d['values']): + if nonempty(d["values"]): return # Add data to path for expr in exprs: self.paths[expr].add(path) - self.data[path].append({ - 'time_info': time_info, - 'values': data - }) + self.data[path].append({"time_info": time_info, "values": data}) def get_series_list(self, path_expr): series_list = [] for path in self.get_paths(path_expr): for data in self.data.get(path): - start, end, step = data['time_info'] - series = TimeSeries(path, start, end, step, data['values']) + start, end, step = data["time_info"] + series = TimeSeries(path, start, end, step, data["values"]) series.pathExpression = path_expr series_list.append(series) return series_list @@ -145,10 +165,11 @@ def get_series_list(self, path_expr): def fetchData(requestContext, pathExprs): from ..app import app - startTime = int(epoch(requestContext['startTime'])) - endTime = int(epoch(requestContext['endTime'])) - if 'now' in requestContext: - now = int(epoch(requestContext['now'])) + + startTime = int(epoch(requestContext["startTime"])) + endTime = int(epoch(requestContext["endTime"])) + if "now" in requestContext: + now = int(epoch(requestContext["now"])) else: now = None @@ -168,7 +189,7 @@ def fetchData(requestContext, pathExprs): if not node.is_leaf: continue if node.path not in path_to_exprs: - if hasattr(node, '__fetch_multi__'): + if hasattr(node, "__fetch_multi__"): multi_nodes[node.__fetch_multi__].append(node) else: single_nodes.append(node) @@ -176,19 +197,19 @@ def fetchData(requestContext, pathExprs): # Multi fetches for finder in app.store.finders: - if not hasattr(finder, '__fetch_multi__'): + if not hasattr(finder, "__fetch_multi__"): continue nodes = multi_nodes[finder.__fetch_multi__] if not nodes: continue try: - time_info, series = finder.fetch_multi(nodes, startTime, endTime, - now, requestContext) + time_info, series = finder.fetch_multi( + nodes, startTime, endTime, now, requestContext + ) except TypeError: time_info, series = finder.fetch_multi(nodes, startTime, endTime) for path, values in series.items(): - data_store.add_data(path, time_info, values, - path_to_exprs[path]) + data_store.add_data(path, time_info, values, path_to_exprs[path]) # Single fetches fetches = [ @@ -197,15 +218,16 @@ def fetchData(requestContext, pathExprs): ] for path, results in fetches: if not results: - logger.info("no results", path=path, start=startTime, - end=endTime) + logger.info("no results", path=path, start=startTime, end=endTime) continue try: time_info, values = results except ValueError as e: - raise Exception("could not parse timeInfo/values from metric " - "'%s': %s" % (path, e)) + raise Exception( + "could not parse timeInfo/values from metric " + "'%s': %s" % (path, e) + ) data_store.add_data(path, time_info, values, path_to_exprs[path]) return data_store diff --git a/graphite_api/render/glyph.py b/graphite_api/render/glyph.py index 3d967f8..26d953e 100644 --- a/graphite_api/render/glyph.py +++ b/graphite_api/render/glyph.py @@ -29,45 +29,47 @@ from ..utils import to_seconds -INFINITY = float('inf') +INFINITY = float("inf") colorAliases = { - 'black': (0, 0, 0), - 'white': (255, 255, 255), - 'blue': (100, 100, 255), - 'green': (0, 200, 0), - 'red': (255, 0, 0), - 'yellow': (255, 255, 0), - 'orange': (255, 165, 0), - 'purple': (200, 100, 255), - 'brown': (150, 100, 50), - 'cyan': (0, 255, 255), - 'aqua': (0, 150, 150), - 'gray': (175, 175, 175), - 'grey': (175, 175, 175), - 'magenta': (255, 0, 255), - 'pink': (255, 100, 100), - 'gold': (200, 200, 0), - 'rose': (200, 150, 200), - 'darkblue': (0, 0, 255), - 'darkgreen': (0, 255, 0), - 'darkred': (200, 00, 50), - 'darkgray': (111, 111, 111), - 'darkgrey': (111, 111, 111), + "black": (0, 0, 0), + "white": (255, 255, 255), + "blue": (100, 100, 255), + "green": (0, 200, 0), + "red": (255, 0, 0), + "yellow": (255, 255, 0), + "orange": (255, 165, 0), + "purple": (200, 100, 255), + "brown": (150, 100, 50), + "cyan": (0, 255, 255), + "aqua": (0, 150, 150), + "gray": (175, 175, 175), + "grey": (175, 175, 175), + "magenta": (255, 0, 255), + "pink": (255, 100, 100), + "gold": (200, 200, 0), + "rose": (200, 150, 200), + "darkblue": (0, 0, 255), + "darkgreen": (0, 255, 0), + "darkred": (200, 00, 50), + "darkgray": (111, 111, 111), + "darkgrey": (111, 111, 111), } # This gets overridden by graphTemplates.conf defaultGraphOptions = dict( - background='white', - foreground='black', - majorline='rose', - minorline='grey', - linecolors=('blue,green,red,purple,brown,yellow,aqua,grey,' - 'magenta,pink,gold,rose'), - fontname='Sans', + background="white", + foreground="black", + majorline="rose", + minorline="grey", + linecolors=( + "blue,green,red,purple,brown,yellow,aqua,grey," + "magenta,pink,gold,rose" + ), + fontname="Sans", fontsize=10, - fontbold='false', - fontitalic='false', + fontbold="false", + fontitalic="false", ) # X-axis configurations (copied from rrdtool, this technique is evil & ugly @@ -81,239 +83,289 @@ YEAR = DAY * 365 xAxisConfigs = ( - dict(seconds=0.00, - minorGridUnit=SEC, - minorGridStep=5, - majorGridUnit=MIN, - majorGridStep=1, - labelUnit=SEC, - labelStep=5, - format="%H:%M:%S", - maxInterval=10*MIN), - dict(seconds=0.07, - minorGridUnit=SEC, - minorGridStep=10, - majorGridUnit=MIN, - majorGridStep=1, - labelUnit=SEC, - labelStep=10, - format="%H:%M:%S", - maxInterval=20*MIN), - dict(seconds=0.14, - minorGridUnit=SEC, - minorGridStep=15, - majorGridUnit=MIN, - majorGridStep=1, - labelUnit=SEC, - labelStep=15, - format="%H:%M:%S", - maxInterval=30*MIN), - dict(seconds=0.27, - minorGridUnit=SEC, - minorGridStep=30, - majorGridUnit=MIN, - majorGridStep=2, - labelUnit=MIN, - labelStep=1, - format="%H:%M", - maxInterval=2*HOUR), - dict(seconds=0.5, - minorGridUnit=MIN, - minorGridStep=1, - majorGridUnit=MIN, - majorGridStep=2, - labelUnit=MIN, - labelStep=1, - format="%H:%M", - maxInterval=2*HOUR), - dict(seconds=1.2, - minorGridUnit=MIN, - minorGridStep=1, - majorGridUnit=MIN, - majorGridStep=4, - labelUnit=MIN, - labelStep=2, - format="%H:%M", - maxInterval=3*HOUR), - dict(seconds=2, - minorGridUnit=MIN, - minorGridStep=1, - majorGridUnit=MIN, - majorGridStep=10, - labelUnit=MIN, - labelStep=5, - format="%H:%M", - maxInterval=6*HOUR), - dict(seconds=5, - minorGridUnit=MIN, - minorGridStep=2, - majorGridUnit=MIN, - majorGridStep=10, - labelUnit=MIN, - labelStep=10, - format="%H:%M", - maxInterval=12*HOUR), - dict(seconds=10, - minorGridUnit=MIN, - minorGridStep=5, - majorGridUnit=MIN, - majorGridStep=20, - labelUnit=MIN, - labelStep=20, - format="%H:%M", - maxInterval=1*DAY), - dict(seconds=30, - minorGridUnit=MIN, - minorGridStep=10, - majorGridUnit=HOUR, - majorGridStep=1, - labelUnit=HOUR, - labelStep=1, - format="%H:%M", - maxInterval=2*DAY), - dict(seconds=60, - minorGridUnit=MIN, - minorGridStep=30, - majorGridUnit=HOUR, - majorGridStep=2, - labelUnit=HOUR, - labelStep=2, - format="%H:%M", - maxInterval=2*DAY), - dict(seconds=100, - minorGridUnit=HOUR, - minorGridStep=2, - majorGridUnit=HOUR, - majorGridStep=4, - labelUnit=HOUR, - labelStep=4, - format="%a %H:%M", - maxInterval=6*DAY), - dict(seconds=255, - minorGridUnit=HOUR, - minorGridStep=6, - majorGridUnit=HOUR, - majorGridStep=12, - labelUnit=HOUR, - labelStep=12, - format="%m/%d %H:%M", - maxInterval=10*DAY), - dict(seconds=600, - minorGridUnit=HOUR, - minorGridStep=6, - majorGridUnit=DAY, - majorGridStep=1, - labelUnit=DAY, - labelStep=1, - format="%m/%d", - maxInterval=14*DAY), - dict(seconds=1000, - minorGridUnit=HOUR, - minorGridStep=12, - majorGridUnit=DAY, - majorGridStep=1, - labelUnit=DAY, - labelStep=1, - format="%m/%d", - maxInterval=365*DAY), - dict(seconds=2000, - minorGridUnit=DAY, - minorGridStep=1, - majorGridUnit=DAY, - majorGridStep=2, - labelUnit=DAY, - labelStep=2, - format="%m/%d", - maxInterval=365*DAY), - dict(seconds=4000, - minorGridUnit=DAY, - minorGridStep=2, - majorGridUnit=DAY, - majorGridStep=4, - labelUnit=DAY, - labelStep=4, - format="%m/%d", - maxInterval=365*DAY), - dict(seconds=8000, - minorGridUnit=DAY, - minorGridStep=3.5, - majorGridUnit=DAY, - majorGridStep=7, - labelUnit=DAY, - labelStep=7, - format="%m/%d", - maxInterval=365*DAY), - dict(seconds=16000, - minorGridUnit=DAY, - minorGridStep=7, - majorGridUnit=DAY, - majorGridStep=14, - labelUnit=DAY, - labelStep=14, - format="%m/%d", - maxInterval=365*DAY), - dict(seconds=32000, - minorGridUnit=DAY, - minorGridStep=15, - majorGridUnit=DAY, - majorGridStep=30, - labelUnit=DAY, - labelStep=30, - format="%m/%d", - maxInterval=365*DAY), - dict(seconds=64000, - minorGridUnit=DAY, - minorGridStep=30, - majorGridUnit=DAY, - majorGridStep=60, - labelUnit=DAY, - labelStep=60, - format="%m/%d %Y"), - dict(seconds=100000, - minorGridUnit=DAY, - minorGridStep=60, - majorGridUnit=DAY, - majorGridStep=120, - labelUnit=DAY, - labelStep=120, - format="%m/%d %Y"), - dict(seconds=120000, - minorGridUnit=DAY, - minorGridStep=120, - majorGridUnit=DAY, - majorGridStep=240, - labelUnit=DAY, - labelStep=240, - format="%m/%d %Y"), + dict( + seconds=0.00, + minorGridUnit=SEC, + minorGridStep=5, + majorGridUnit=MIN, + majorGridStep=1, + labelUnit=SEC, + labelStep=5, + format="%H:%M:%S", + maxInterval=10 * MIN, + ), + dict( + seconds=0.07, + minorGridUnit=SEC, + minorGridStep=10, + majorGridUnit=MIN, + majorGridStep=1, + labelUnit=SEC, + labelStep=10, + format="%H:%M:%S", + maxInterval=20 * MIN, + ), + dict( + seconds=0.14, + minorGridUnit=SEC, + minorGridStep=15, + majorGridUnit=MIN, + majorGridStep=1, + labelUnit=SEC, + labelStep=15, + format="%H:%M:%S", + maxInterval=30 * MIN, + ), + dict( + seconds=0.27, + minorGridUnit=SEC, + minorGridStep=30, + majorGridUnit=MIN, + majorGridStep=2, + labelUnit=MIN, + labelStep=1, + format="%H:%M", + maxInterval=2 * HOUR, + ), + dict( + seconds=0.5, + minorGridUnit=MIN, + minorGridStep=1, + majorGridUnit=MIN, + majorGridStep=2, + labelUnit=MIN, + labelStep=1, + format="%H:%M", + maxInterval=2 * HOUR, + ), + dict( + seconds=1.2, + minorGridUnit=MIN, + minorGridStep=1, + majorGridUnit=MIN, + majorGridStep=4, + labelUnit=MIN, + labelStep=2, + format="%H:%M", + maxInterval=3 * HOUR, + ), + dict( + seconds=2, + minorGridUnit=MIN, + minorGridStep=1, + majorGridUnit=MIN, + majorGridStep=10, + labelUnit=MIN, + labelStep=5, + format="%H:%M", + maxInterval=6 * HOUR, + ), + dict( + seconds=5, + minorGridUnit=MIN, + minorGridStep=2, + majorGridUnit=MIN, + majorGridStep=10, + labelUnit=MIN, + labelStep=10, + format="%H:%M", + maxInterval=12 * HOUR, + ), + dict( + seconds=10, + minorGridUnit=MIN, + minorGridStep=5, + majorGridUnit=MIN, + majorGridStep=20, + labelUnit=MIN, + labelStep=20, + format="%H:%M", + maxInterval=1 * DAY, + ), + dict( + seconds=30, + minorGridUnit=MIN, + minorGridStep=10, + majorGridUnit=HOUR, + majorGridStep=1, + labelUnit=HOUR, + labelStep=1, + format="%H:%M", + maxInterval=2 * DAY, + ), + dict( + seconds=60, + minorGridUnit=MIN, + minorGridStep=30, + majorGridUnit=HOUR, + majorGridStep=2, + labelUnit=HOUR, + labelStep=2, + format="%H:%M", + maxInterval=2 * DAY, + ), + dict( + seconds=100, + minorGridUnit=HOUR, + minorGridStep=2, + majorGridUnit=HOUR, + majorGridStep=4, + labelUnit=HOUR, + labelStep=4, + format="%a %H:%M", + maxInterval=6 * DAY, + ), + dict( + seconds=255, + minorGridUnit=HOUR, + minorGridStep=6, + majorGridUnit=HOUR, + majorGridStep=12, + labelUnit=HOUR, + labelStep=12, + format="%m/%d %H:%M", + maxInterval=10 * DAY, + ), + dict( + seconds=600, + minorGridUnit=HOUR, + minorGridStep=6, + majorGridUnit=DAY, + majorGridStep=1, + labelUnit=DAY, + labelStep=1, + format="%m/%d", + maxInterval=14 * DAY, + ), + dict( + seconds=1000, + minorGridUnit=HOUR, + minorGridStep=12, + majorGridUnit=DAY, + majorGridStep=1, + labelUnit=DAY, + labelStep=1, + format="%m/%d", + maxInterval=365 * DAY, + ), + dict( + seconds=2000, + minorGridUnit=DAY, + minorGridStep=1, + majorGridUnit=DAY, + majorGridStep=2, + labelUnit=DAY, + labelStep=2, + format="%m/%d", + maxInterval=365 * DAY, + ), + dict( + seconds=4000, + minorGridUnit=DAY, + minorGridStep=2, + majorGridUnit=DAY, + majorGridStep=4, + labelUnit=DAY, + labelStep=4, + format="%m/%d", + maxInterval=365 * DAY, + ), + dict( + seconds=8000, + minorGridUnit=DAY, + minorGridStep=3.5, + majorGridUnit=DAY, + majorGridStep=7, + labelUnit=DAY, + labelStep=7, + format="%m/%d", + maxInterval=365 * DAY, + ), + dict( + seconds=16000, + minorGridUnit=DAY, + minorGridStep=7, + majorGridUnit=DAY, + majorGridStep=14, + labelUnit=DAY, + labelStep=14, + format="%m/%d", + maxInterval=365 * DAY, + ), + dict( + seconds=32000, + minorGridUnit=DAY, + minorGridStep=15, + majorGridUnit=DAY, + majorGridStep=30, + labelUnit=DAY, + labelStep=30, + format="%m/%d", + maxInterval=365 * DAY, + ), + dict( + seconds=64000, + minorGridUnit=DAY, + minorGridStep=30, + majorGridUnit=DAY, + majorGridStep=60, + labelUnit=DAY, + labelStep=60, + format="%m/%d %Y", + ), + dict( + seconds=100000, + minorGridUnit=DAY, + minorGridStep=60, + majorGridUnit=DAY, + majorGridStep=120, + labelUnit=DAY, + labelStep=120, + format="%m/%d %Y", + ), + dict( + seconds=120000, + minorGridUnit=DAY, + minorGridStep=120, + majorGridUnit=DAY, + majorGridStep=240, + labelUnit=DAY, + labelStep=240, + format="%m/%d %Y", + ), ) UnitSystems = { - 'binary': ( - ('Pi', 1024.0**5), - ('Ti', 1024.0**4), - ('Gi', 1024.0**3), - ('Mi', 1024.0**2), - ('Ki', 1024.0)), - 'si': ( - ('P', 1000.0**5), - ('T', 1000.0**4), - ('G', 1000.0**3), - ('M', 1000.0**2), - ('K', 1000.0)), - 'sec': ( - ('Y', 60*60*24*365), - ('M', 60*60*24*30), - ('D', 60*60*24), - ('H', 60*60), - ('m', 60)), - 'msec': ( - ('Y', 60*60*24*365*1000), - ('M', 60*60*24*30*1000), - ('D', 60*60*24*1000), - ('H', 60*60*1000), - ('m', 60*1000), - ('s', 1000)), - 'none': [], + "binary": ( + ("Pi", 1024.0 ** 5), + ("Ti", 1024.0 ** 4), + ("Gi", 1024.0 ** 3), + ("Mi", 1024.0 ** 2), + ("Ki", 1024.0), + ), + "si": ( + ("P", 1000.0 ** 5), + ("T", 1000.0 ** 4), + ("G", 1000.0 ** 3), + ("M", 1000.0 ** 2), + ("K", 1000.0), + ), + "sec": ( + ("Y", 60 * 60 * 24 * 365), + ("M", 60 * 60 * 24 * 30), + ("D", 60 * 60 * 24), + ("H", 60 * 60), + ("m", 60), + ), + "msec": ( + ("Y", 60 * 60 * 24 * 365 * 1000), + ("M", 60 * 60 * 24 * 30 * 1000), + ("D", 60 * 60 * 24 * 1000), + ("H", 60 * 60 * 1000), + ("m", 60 * 1000), + ("s", 1000), + ), + "none": [], } @@ -338,22 +390,22 @@ class GraphError(Exception): class _AxisTics: def __init__(self, minValue, maxValue, unitSystem=None): self.minValue = self.checkFinite(minValue, "data value") - self.minValueSource = 'data' + self.minValueSource = "data" self.maxValue = self.checkFinite(maxValue, "data value") - self.maxValueSource = 'data' + self.maxValueSource = "data" self.unitSystem = unitSystem @staticmethod - def checkFinite(value, name='value'): + def checkFinite(value, name="value"): """Check that value is a finite number. If it is, return it. If not, raise GraphError describing the problem, using name in the error message. """ if math.isnan(value): - raise GraphError('Encountered NaN %s' % (name,)) + raise GraphError("Encountered NaN %s" % (name,)) elif math.isinf(value): - raise GraphError('Encountered infinite %s' % (name,)) + raise GraphError("Encountered infinite %s" % (name,)) return value @staticmethod @@ -381,19 +433,22 @@ def reconcileLimits(self): # The limits are already OK. return - minFixed = (self.minValueSource in ['min']) - maxFixed = (self.maxValueSource in ['max', 'limit']) + minFixed = self.minValueSource in ["min"] + maxFixed = self.maxValueSource in ["max", "limit"] if minFixed and maxFixed: - raise GraphError('The %s must be less than the %s' % - (self.minValueSource, self.maxValueSource)) + raise GraphError( + "The %s must be less than the %s" + % (self.minValueSource, self.maxValueSource) + ) elif minFixed: self.maxValue = self.minValue + self.chooseDelta(self.minValue) elif maxFixed: self.minValue = self.maxValue - self.chooseDelta(self.maxValue) else: - delta = self.chooseDelta(max(abs(self.minValue), - abs(self.maxValue))) + delta = self.chooseDelta( + max(abs(self.minValue), abs(self.maxValue)) + ) average = (self.minValue + self.maxValue) / 2.0 self.minValue = average - delta self.maxValue = average + delta @@ -418,20 +473,20 @@ def applySettings(self, axisMin=None, axisMax=None, axisLimit=None): or None to impose no upper limit. """ if axisMin is not None and not math.isnan(axisMin): - self.minValueSource = 'min' - self.minValue = self.checkFinite(axisMin, 'axis min') + self.minValueSource = "min" + self.minValue = self.checkFinite(axisMin, "axis min") - if axisMax == 'max': - self.maxValueSource = 'extremum' + if axisMax == "max": + self.maxValueSource = "extremum" elif axisMax is not None and not math.isnan(axisMax): - self.maxValueSource = 'max' - self.maxValue = self.checkFinite(axisMax, 'axis max') + self.maxValueSource = "max" + self.maxValue = self.checkFinite(axisMax, "axis max") if axisLimit is None or math.isnan(axisLimit): self.axisLimit = None elif axisLimit < self.maxValue: - self.maxValue = self.checkFinite(axisLimit, 'axis limit') - self.maxValueSource = 'limit' + self.maxValue = self.checkFinite(axisLimit, "axis limit") + self.maxValueSource = "limit" # The limit has already been imposed, so there is no need to # remember it: self.axisLimit = None @@ -451,10 +506,10 @@ def makeLabel(self, value): Create a label string containing the value and its units (if any), based on the values of self.step, self.span, and self.unitSystem. """ - value, prefix = format_units(value, self.step, - system=self.unitSystem) - span, spanPrefix = format_units(self.span, self.step, - system=self.unitSystem) + value, prefix = format_units(value, self.step, system=self.unitSystem) + span, spanPrefix = format_units( + self.span, self.step, system=self.unitSystem + ) if prefix: prefix += " " if value < 0.1: @@ -485,7 +540,7 @@ def __init__(self, minValue, maxValue, unitSystem=None): def setStep(self, step): """Set the size of steps between ticmarks.""" - self.step = self.checkFinite(float(step), 'axis step') + self.step = self.checkFinite(float(step), "axis step") def generateSteps(self, minStep): """Generate allowed steps with step >= minStep in increasing order.""" @@ -570,10 +625,11 @@ def chooseStep(self, divisors=None, binary=False): divisors = [4, 5, 6] else: for divisor in divisors: - self.checkFinite(divisor, 'divisor') + self.checkFinite(divisor, "divisor") if divisor < 1: - raise GraphError('Divisors must be greater than or equal ' - 'to one') + raise GraphError( + "Divisors must be greater than or equal " "to one" + ) if self.minValue == self.maxValue: if self.minValue == 0.0: @@ -591,8 +647,8 @@ def chooseStep(self, divisors=None, binary=False): bestStep = None for step in self.generateSteps(variance / float(max(divisors))): if ( - bestSlop is not None and - step * min(divisors) >= 2 * bestSlop + variance + bestSlop is not None + and step * min(divisors) >= 2 * bestSlop + variance ): break for divisor in divisors: @@ -604,22 +660,24 @@ def chooseStep(self, divisors=None, binary=False): self.step = bestStep def chooseLimits(self): - if self.minValueSource == 'data': + if self.minValueSource == "data": # Start labels at the greatest multiple of step <= minValue: self.bottom = self.step * math.floor( - (self.minValue / self.step + EPSILON)) + (self.minValue / self.step + EPSILON) + ) else: self.bottom = self.minValue - if self.maxValueSource == 'data': + if self.maxValueSource == "data": # Extend the top of our graph to the lowest # step multiple >= maxValue: self.top = self.step * math.ceil( - (self.maxValue / self.step - EPSILON)) + (self.maxValue / self.step - EPSILON) + ) # ...but never exceed a user-specified limit: if ( - self.axisLimit is not None and - self.top > self.axisLimit + EPSILON * self.step + self.axisLimit is not None + and self.top > self.axisLimit + EPSILON * self.step ): self.top = self.axisLimit else: @@ -633,7 +691,7 @@ def chooseLimits(self): def getLabelValues(self): if self.step <= 0.0: - raise GraphError('The step size must be positive') + raise GraphError("The step size must be positive") if self.span > 1000.0 * self.step: # This is insane. Pick something that won't cause trouble: self.chooseStep() @@ -656,8 +714,8 @@ class _LogAxisTics(_AxisTics): def __init__(self, minValue, maxValue, unitSystem=None, base=10.0): _AxisTics.__init__(self, minValue, maxValue, unitSystem=unitSystem) if base <= 1.0: - raise GraphError('Logarithmic base must be greater than one') - self.base = self.checkFinite(base, 'log base') + raise GraphError("Logarithmic base must be greater than one") + self.base = self.checkFinite(base, "log base") self.step = None self.span = None @@ -671,12 +729,16 @@ def chooseStep(self, divisors=None, binary=False): def chooseLimits(self): if self.minValue <= 0: - raise GraphError('Logarithmic scale specified with a dataset with ' - 'a minimum value less than or equal to zero') - self.bottom = math.pow(self.base, - math.floor(math.log(self.minValue, self.base))) - self.top = math.pow(self.base, - math.ceil(math.log(self.maxValue, self.base))) + raise GraphError( + "Logarithmic scale specified with a dataset with " + "a minimum value less than or equal to zero" + ) + self.bottom = math.pow( + self.base, math.floor(math.log(self.minValue, self.base)) + ) + self.top = math.pow( + self.base, math.ceil(math.log(self.maxValue, self.base)) + ) self.span = self.top - self.bottom @@ -687,8 +749,9 @@ def chooseLimits(self): def getLabelValues(self): values = [] - value = math.pow(self.base, - math.ceil(math.log(self.bottom, self.base) - EPSILON)) + value = math.pow( + self.base, math.ceil(math.log(self.bottom, self.base) - EPSILON) + ) while value < self.top * (1.0 + EPSILON): values.append(value) value *= self.base @@ -697,77 +760,92 @@ def getLabelValues(self): class Graph(object): - customizable = ('width', 'height', 'margin', 'bgcolor', 'fgcolor', - 'fontName', 'fontSize', 'fontBold', 'fontItalic', - 'colorList', 'template', 'yAxisSide', 'outputFormat') + customizable = ( + "width", + "height", + "margin", + "bgcolor", + "fgcolor", + "fontName", + "fontSize", + "fontBold", + "fontItalic", + "colorList", + "template", + "yAxisSide", + "outputFormat", + ) def __init__(self, **params): self.params = params - self.data = params['data'] + self.data = params["data"] self.dataLeft = [] self.dataRight = [] self.secondYAxis = False - self.width = int(params.get('width', 200)) - self.height = int(params.get('height', 200)) - self.margin = int(params.get('margin', 10)) - self.userTimeZone = params.get('tz') - self.logBase = params.get('logBase', None) - self.minorY = int(params.get('minorY', 1)) + self.width = int(params.get("width", 200)) + self.height = int(params.get("height", 200)) + self.margin = int(params.get("margin", 10)) + self.userTimeZone = params.get("tz") + self.logBase = params.get("logBase", None) + self.minorY = int(params.get("minorY", 1)) if self.logBase: - if self.logBase == 'e': + if self.logBase == "e": self.logBase = math.e elif self.logBase <= 1: self.logBase = None - params['logBase'] = None + params["logBase"] = None else: self.logBase = float(self.logBase) if self.margin < 0: self.margin = 10 - self.setupCairo(params.get('outputFormat', 'png').lower()) + self.setupCairo(params.get("outputFormat", "png").lower()) self.area = { - 'xmin': self.margin + 10, # Need extra room when the time is - # near the left edge - 'xmax': self.width - self.margin, - 'ymin': self.margin, - 'ymax': self.height - self.margin, + "xmin": self.margin + 10, # Need extra room when the time is + # near the left edge + "xmax": self.width - self.margin, + "ymin": self.margin, + "ymax": self.height - self.margin, } - self.loadTemplate(params.get('template', 'default')) + self.loadTemplate(params.get("template", "default")) opts = self.ctx.get_font_options() opts.set_antialias(cairo.ANTIALIAS_NONE) self.ctx.set_font_options(opts) - self.foregroundColor = params.get('fgcolor', self.defaultForeground) - self.backgroundColor = params.get('bgcolor', self.defaultBackground) + self.foregroundColor = params.get("fgcolor", self.defaultForeground) + self.backgroundColor = params.get("bgcolor", self.defaultBackground) self.setColor(self.backgroundColor) self.drawRectangle(0, 0, self.width, self.height) - if 'colorList' in params: - colorList = unquote_plus(str(params['colorList'])).split(',') + if "colorList" in params: + colorList = unquote_plus(str(params["colorList"])).split(",") else: colorList = self.defaultColorList self.colors = itertools.cycle(colorList) self.drawGraph(**params) - def setupCairo(self, outputFormat='png'): + def setupCairo(self, outputFormat="png"): self.outputFormat = outputFormat - if outputFormat == 'png': - self.surface = cairo.ImageSurface(cairo.FORMAT_ARGB32, - self.width, self.height) - elif outputFormat == 'svg': + if outputFormat == "png": + self.surface = cairo.ImageSurface( + cairo.FORMAT_ARGB32, self.width, self.height + ) + elif outputFormat == "svg": self.surfaceData = BytesIO() - self.surface = cairo.SVGSurface(self.surfaceData, - self.width, self.height) - elif outputFormat == 'pdf': + self.surface = cairo.SVGSurface( + self.surfaceData, self.width, self.height + ) + elif outputFormat == "pdf": self.surfaceData = BytesIO() - self.surface = cairo.PDFSurface(self.surfaceData, - self.width, self.height) + self.surface = cairo.PDFSurface( + self.surfaceData, self.width, self.height + ) res_x, res_y = self.surface.get_fallback_resolution() self.width = float(self.width / res_x) * 72 self.height = float(self.height / res_y) * 72 @@ -781,37 +859,45 @@ def setColor(self, value, alpha=1.0, forceAlpha=False): r, g, b = colorAliases[value] elif isinstance(value, six.string_types) and len(value) >= 6: s = value - if s.startswith('#'): + if s.startswith("#"): s = s[1:] - if s.startswith('%23'): + if s.startswith("%23"): s = s[3:] - r, g, b = (int(s[0:2], base=16), int(s[2:4], base=16), - int(s[4:6], base=16)) + r, g, b = ( + int(s[0:2], base=16), + int(s[2:4], base=16), + int(s[4:6], base=16), + ) if len(s) == 8 and not forceAlpha: alpha = int(s[6:8], base=16) / 255.0 elif isinstance(value, int) and len(str(value)) == 6: s = str(value) - r, g, b = (int(s[0:2], base=16), int(s[2:4], base=16), - int(s[4:6], base=16)) + r, g, b = ( + int(s[0:2], base=16), + int(s[2:4], base=16), + int(s[4:6], base=16), + ) else: - raise ValueError("Must specify an RGB 3-tuple, an html color " - "string, or a known color alias!") + raise ValueError( + "Must specify an RGB 3-tuple, an html color " + "string, or a known color alias!" + ) r, g, b = [float(c) / 255.0 for c in (r, g, b)] self.ctx.set_source_rgba(r, g, b, alpha) def setFont(self, **params): p = self.defaultFontParams.copy() p.update(params) - self.ctx.select_font_face(p['name'], p['italic'], p['bold']) - self.ctx.set_font_size(float(p['size'])) + self.ctx.select_font_face(p["name"], p["italic"], p["bold"]) + self.ctx.set_font_size(float(p["size"])) def getExtents(self, text=None): F = self.ctx.font_extents() - extents = {'maxHeight': F[2], 'maxAscent': F[0], 'maxDescent': F[1]} + extents = {"maxHeight": F[2], "maxAscent": F[0], "maxDescent": F[1]} if text is not None: T = self.ctx.text_extents(text) - extents['width'] = T[4] - extents['height'] = T[3] + extents["width"] = T[4] + extents["height"] = T[3] return extents def drawRectangle(self, x, y, w, h, fill=True): @@ -829,51 +915,52 @@ def drawRectangle(self, x, y, w, h, fill=True): self.ctx.set_dash([], 0) self.ctx.stroke() - def drawText(self, text, x, y, align='left', valign='top', rotate=0): + def drawText(self, text, x, y, align="left", valign="top", rotate=0): extents = self.getExtents(text) angle = math.radians(rotate) origMatrix = self.ctx.get_matrix() horizontal = { - 'left': 0, - 'center': extents['width'] / 2, - 'right': extents['width'], + "left": 0, + "center": extents["width"] / 2, + "right": extents["width"], }[align.lower()] vertical = { - 'top': extents['maxAscent'], - 'middle': extents['maxHeight'] / 2 - extents['maxDescent'], - 'bottom': -extents['maxDescent'], - 'baseline': 0, + "top": extents["maxAscent"], + "middle": extents["maxHeight"] / 2 - extents["maxDescent"], + "bottom": -extents["maxDescent"], + "baseline": 0, }[valign.lower()] self.ctx.move_to(x, y) - self.ctx.rel_move_to(math.sin(angle) * -vertical, - math.cos(angle) * vertical) + self.ctx.rel_move_to( + math.sin(angle) * -vertical, math.cos(angle) * vertical + ) self.ctx.rotate(angle) self.ctx.rel_move_to(-horizontal, 0) bx, by = self.ctx.get_current_point() - by -= extents['maxAscent'] + by -= extents["maxAscent"] self.ctx.text_path(text) self.ctx.fill() self.ctx.set_matrix(origMatrix) def drawTitle(self, text): - self.encodeHeader('title') + self.encodeHeader("title") - y = self.area['ymin'] + y = self.area["ymin"] x = self.width / 2 - lineHeight = self.getExtents()['maxHeight'] - for line in text.split('\n'): - self.drawText(line, x, y, align='center') + lineHeight = self.getExtents()["maxHeight"] + for line in text.split("\n"): + self.drawText(line, x, y, align="center") y += lineHeight - if self.params.get('yAxisSide') == 'right': - self.area['ymin'] = y + if self.params.get("yAxisSide") == "right": + self.area["ymin"] = y else: - self.area['ymin'] = y + self.margin + self.area["ymin"] = y + self.margin def drawLegend(self, elements, unique=False): # elements is [ (name,color,rightSide), (name,color,rightSide), ... ] - self.encodeHeader('legend') + self.encodeHeader("legend") if unique: # remove duplicate names @@ -892,97 +979,113 @@ def drawLegend(self, elements, unique=False): # Double it to check if there's enough room for 2 columns testSizeName = longestName + " " + longestName testExt = self.getExtents(testSizeName) - testBoxSize = testExt['maxHeight'] - 1 - testWidth = testExt['width'] + 2 * (testBoxSize + padding) + testBoxSize = testExt["maxHeight"] - 1 + testWidth = testExt["width"] + 2 * (testBoxSize + padding) if testWidth + 50 < self.width: rightSideLabels = True if self.secondYAxis and rightSideLabels: extents = self.getExtents(longestName) padding = 5 - boxSize = extents['maxHeight'] - 1 - lineHeight = extents['maxHeight'] + 1 - labelWidth = extents['width'] + 2 * (boxSize + padding) - columns = max(1, math.floor( - (self.width - self.area['xmin']) / labelWidth)) - numRight = len([name for (name, color, rightSide) in elements - if rightSide]) + boxSize = extents["maxHeight"] - 1 + lineHeight = extents["maxHeight"] + 1 + labelWidth = extents["width"] + 2 * (boxSize + padding) + columns = max( + 1, math.floor((self.width - self.area["xmin"]) / labelWidth) + ) + numRight = len( + [name for (name, color, rightSide) in elements if rightSide] + ) numberOfLines = max(len(elements) - numRight, numRight) columns = math.floor(columns / 2.0) columns = max(columns, 1) legendHeight = ( - max(1, (numberOfLines / columns)) * lineHeight) + padding + max(1, (numberOfLines / columns)) * lineHeight + ) + padding # scoot the drawing area up to fit the legend - self.area['ymax'] -= legendHeight + self.area["ymax"] -= legendHeight self.ctx.set_line_width(1.0) - x = self.area['xmin'] - y = self.area['ymax'] + (2 * padding) + x = self.area["xmin"] + y = self.area["ymax"] + (2 * padding) n = 0 - xRight = self.area['xmax'] - self.area['xmin'] + xRight = self.area["xmax"] - self.area["xmin"] yRight = y nRight = 0 for name, color, rightSide in elements: self.setColor(color) if rightSide: nRight += 1 - self.drawRectangle(xRight - padding, yRight, - boxSize, boxSize) - self.setColor('darkgrey') - self.drawRectangle(xRight - padding, yRight, - boxSize, boxSize, fill=False) + self.drawRectangle( + xRight - padding, yRight, boxSize, boxSize + ) + self.setColor("darkgrey") + self.drawRectangle( + xRight - padding, yRight, boxSize, boxSize, fill=False + ) self.setColor(self.foregroundColor) - self.drawText(name, xRight - boxSize, yRight, - align='right') + self.drawText( + name, xRight - boxSize, yRight, align="right" + ) xRight -= labelWidth if nRight % columns == 0: - xRight = self.area['xmax'] - self.area['xmin'] + xRight = self.area["xmax"] - self.area["xmin"] yRight += lineHeight else: n += 1 self.drawRectangle(x, y, boxSize, boxSize) - self.setColor('darkgrey') + self.setColor("darkgrey") self.drawRectangle(x, y, boxSize, boxSize, fill=False) self.setColor(self.foregroundColor) - self.drawText(name, x + boxSize + padding, y, align='left') + self.drawText( + name, x + boxSize + padding, y, align="left" + ) x += labelWidth if n % columns == 0: - x = self.area['xmin'] + x = self.area["xmin"] y += lineHeight else: extents = self.getExtents(longestName) - boxSize = extents['maxHeight'] - 1 - lineHeight = extents['maxHeight'] + 1 - labelWidth = extents['width'] + 2 * (boxSize + padding) + boxSize = extents["maxHeight"] - 1 + lineHeight = extents["maxHeight"] + 1 + labelWidth = extents["width"] + 2 * (boxSize + padding) columns = math.floor(self.width / labelWidth) columns = max(columns, 1) numberOfLines = math.ceil(float(len(elements)) / columns) legendHeight = (numberOfLines * lineHeight) + padding # scoot the drawing area up to fit the legend - self.area['ymax'] -= legendHeight + self.area["ymax"] -= legendHeight self.ctx.set_line_width(1.0) - x = self.area['xmin'] - y = self.area['ymax'] + (2 * padding) + x = self.area["xmin"] + y = self.area["ymax"] + (2 * padding) for i, (name, color, rightSide) in enumerate(elements): if rightSide: self.setColor(color) - self.drawRectangle(x + labelWidth + padding, y, - boxSize, boxSize) - self.setColor('darkgrey') - self.drawRectangle(x + labelWidth + padding, y, - boxSize, boxSize, fill=False) + self.drawRectangle( + x + labelWidth + padding, y, boxSize, boxSize + ) + self.setColor("darkgrey") + self.drawRectangle( + x + labelWidth + padding, + y, + boxSize, + boxSize, + fill=False, + ) self.setColor(self.foregroundColor) - self.drawText(name, x + labelWidth, y, align='right') + self.drawText(name, x + labelWidth, y, align="right") x += labelWidth else: self.setColor(color) self.drawRectangle(x, y, boxSize, boxSize) - self.setColor('darkgrey') + self.setColor("darkgrey") self.drawRectangle(x, y, boxSize, boxSize, fill=False) self.setColor(self.foregroundColor) - self.drawText(name, x + boxSize + padding, y, align='left') + self.drawText( + name, x + boxSize + padding, y, align="left" + ) x += labelWidth if (i + 1) % columns == 0: - x = self.area['xmin'] + x = self.area["xmin"] y += lineHeight def encodeHeader(self, text): @@ -990,79 +1093,86 @@ def encodeHeader(self, text): self.setColor(self.backgroundColor) self.ctx.move_to(-88, -88) # identifier for i, char in enumerate(text): - self.ctx.line_to(-ord(char), -i-1) + self.ctx.line_to(-ord(char), -i - 1) self.ctx.stroke() self.ctx.restore() def loadTemplate(self, template): opts = defaults = defaultGraphOptions - self.defaultBackground = opts.get('background', defaults['background']) - self.defaultForeground = opts.get('foreground', defaults['foreground']) - self.defaultMajorGridLineColor = opts.get('majorline', - defaults['majorline']) - self.defaultMinorGridLineColor = opts.get('minorline', - defaults['minorline']) + self.defaultBackground = opts.get( + "background", defaults["background"] + ) + self.defaultForeground = opts.get( + "foreground", defaults["foreground"] + ) + self.defaultMajorGridLineColor = opts.get( + "majorline", defaults["majorline"] + ) + self.defaultMinorGridLineColor = opts.get( + "minorline", defaults["minorline"] + ) self.defaultColorList = [ - c.strip() for c in opts.get('linecolors', - defaults['linecolors']).split(',')] - fontName = opts.get('fontname', defaults['fontname']) - fontSize = float(opts.get('fontsize', defaults['fontsize'])) - fontBold = opts.get('fontbold', defaults['fontbold']).lower() == 'true' - fontItalic = opts.get('fontitalic', - defaults['fontitalic']).lower() == 'true' + c.strip() + for c in opts.get("linecolors", defaults["linecolors"]).split(",") + ] + fontName = opts.get("fontname", defaults["fontname"]) + fontSize = float(opts.get("fontsize", defaults["fontsize"])) + fontBold = ( + opts.get("fontbold", defaults["fontbold"]).lower() == "true" + ) + fontItalic = ( + opts.get("fontitalic", defaults["fontitalic"]).lower() == "true" + ) self.defaultFontParams = { - 'name': self.params.get('fontName', fontName), - 'size': int(self.params.get('fontSize', fontSize)), - 'bold': self.params.get('fontBold', fontBold), - 'italic': self.params.get('fontItalic', fontItalic), + "name": self.params.get("fontName", fontName), + "size": int(self.params.get("fontSize", fontSize)), + "bold": self.params.get("fontBold", fontBold), + "italic": self.params.get("fontItalic", fontItalic), } def output(self, fileObj): - if self.outputFormat == 'png': + if self.outputFormat == "png": self.surface.write_to_png(fileObj) - elif self.outputFormat == 'pdf': + elif self.outputFormat == "pdf": self.surface.finish() pdfData = self.surfaceData.getvalue() self.surfaceData.close() fileObj.write(pdfData) else: - if hasattr(self, 'startTime'): + if hasattr(self, "startTime"): has_data = True metaData = { - 'x': { - 'start': self.startTime, - 'end': self.endTime - }, - 'options': { - 'lineWidth': self.lineWidth - }, - 'font': self.defaultFontParams, - 'area': self.area, - 'series': [] + "x": {"start": self.startTime, "end": self.endTime}, + "options": {"lineWidth": self.lineWidth}, + "font": self.defaultFontParams, + "area": self.area, + "series": [], } if not self.secondYAxis: - metaData['y'] = { - 'top': self.yTop, - 'bottom': self.yBottom, - 'step': self.yStep, - 'labels': self.yLabels, - 'labelValues': self.yLabelValues + metaData["y"] = { + "top": self.yTop, + "bottom": self.yBottom, + "step": self.yStep, + "labels": self.yLabels, + "labelValues": self.yLabelValues, } for series in self.data: - if 'stacked' not in series.options: - metaData['series'].append({ - 'name': series.name, - 'start': series.start, - 'end': series.end, - 'step': series.step, - 'valuesPerPoint': series.valuesPerPoint, - 'color': series.color, - 'data': series, - 'options': series.options - }) + if "stacked" not in series.options: + metaData["series"].append( + { + "name": series.name, + "start": series.start, + "end": series.end, + "step": series.step, + "valuesPerPoint": series.valuesPerPoint, + "color": series.color, + "data": series, + "options": series.options, + } + ) else: has_data = False metaData = {} @@ -1073,53 +1183,102 @@ def output(self, fileObj): # we expect height/width in pixels, not points svgData = svgData.decode().replace('pt"', 'px"', 2) - svgData = svgData.replace('\n', '', 1) - svgData = svgData.replace('\n\n\n", "", 1) + svgData = svgData.replace( + "\n\n wrappers instead def onHeaderPath(match): - name = '' - for char in re.findall(r'L -(\d+) -\d+', match.group(1)): + name = "" + for char in re.findall(r"L -(\d+) -\d+", match.group(1)): name += chr(int(char)) return '' % name - (svgData, subsMade) = re.subn(r'', - onHeaderPath, svgData) + + (svgData, subsMade) = re.subn( + r'', onHeaderPath, svgData + ) # Replace the first with , and close out the # last at the end - svgData = svgData.replace(' 0: svgData += "" - svgData = svgData.replace(' data-header="true"', '') + svgData = svgData.replace(' data-header="true"', "") fileObj.write(svgData.encode()) - fileObj.write((""" -""" % json.dumps(metaData)).encode()) +""" + % json.dumps(metaData) + ).encode() + ) class LineGraph(Graph): customizable = Graph.customizable + ( - 'title', 'vtitle', 'lineMode', 'lineWidth', 'hideLegend', 'hideAxes', - 'minXStep', 'hideGrid', 'majorGridLineColor', 'minorGridLineColor', - 'thickness', 'min', 'max', 'graphOnly', 'yMin', 'yMax', 'yLimit', - 'yStep', 'areaMode', 'areaAlpha', 'drawNullAsZero', 'tz', 'yAxisSide', - 'pieMode', 'yUnitSystem', 'logBase', 'yMinLeft', 'yMinRight', - 'yMaxLeft', 'yMaxRight', 'yLimitLeft', 'yLimitRight', 'yStepLeft', - 'yStepRight', 'rightWidth', 'rightColor', 'rightDashed', 'leftWidth', - 'leftColor', 'leftDashed', 'xFormat', 'minorY', 'hideYAxis', - 'hideXAxis', 'uniqueLegend', 'vtitleRight', 'yDivisors', - 'connectedLimit', 'hideNullFromLegend') - validLineModes = ('staircase', 'slope', 'connected') - validAreaModes = ('none', 'first', 'all', 'stacked') - validPieModes = ('maximum', 'minimum', 'average') + "title", + "vtitle", + "lineMode", + "lineWidth", + "hideLegend", + "hideAxes", + "minXStep", + "hideGrid", + "majorGridLineColor", + "minorGridLineColor", + "thickness", + "min", + "max", + "graphOnly", + "yMin", + "yMax", + "yLimit", + "yStep", + "areaMode", + "areaAlpha", + "drawNullAsZero", + "tz", + "yAxisSide", + "pieMode", + "yUnitSystem", + "logBase", + "yMinLeft", + "yMinRight", + "yMaxLeft", + "yMaxRight", + "yLimitLeft", + "yLimitRight", + "yStepLeft", + "yStepRight", + "rightWidth", + "rightColor", + "rightDashed", + "leftWidth", + "leftColor", + "leftDashed", + "xFormat", + "minorY", + "hideYAxis", + "hideXAxis", + "uniqueLegend", + "vtitleRight", + "yDivisors", + "connectedLimit", + "hideNullFromLegend", + ) + validLineModes = ("staircase", "slope", "connected") + validAreaModes = ("none", "first", "all", "stacked") + validPieModes = ("maximum", "minimum", "average") def drawGraph(self, **params): # Make sure we've got datapoints to draw @@ -1133,14 +1292,14 @@ def drawGraph(self, **params): if not timeRange: x = self.width / 2 y = self.height / 2 - self.setColor('red') + self.setColor("red") self.setFont(size=math.log(self.width * self.height)) - self.drawText("No Data", x, y, align='center') + self.drawText("No Data", x, y, align="center") return # Determine if we're doing a 2 y-axis graph. for series in self.data: - if 'secondYAxis' in series.options: + if "secondYAxis" in series.options: self.dataRight.append(series) else: self.dataLeft.append(series) @@ -1148,139 +1307,146 @@ def drawGraph(self, **params): self.secondYAxis = True # API compatibilty hacks - if params.get('graphOnly', False): - params['hideLegend'] = True - params['hideGrid'] = True - params['hideAxes'] = True - params['hideXAxis'] = False - params['hideYAxis'] = False - params['yAxisSide'] = 'left' - params['title'] = '' - params['vtitle'] = '' - params['margin'] = 0 - params['tz'] = '' + if params.get("graphOnly", False): + params["hideLegend"] = True + params["hideGrid"] = True + params["hideAxes"] = True + params["hideXAxis"] = False + params["hideYAxis"] = False + params["yAxisSide"] = "left" + params["title"] = "" + params["vtitle"] = "" + params["margin"] = 0 + params["tz"] = "" self.margin = 0 - self.area['xmin'] = 0 - self.area['xmax'] = self.width - self.area['ymin'] = 0 - self.area['ymax'] = self.height - if 'yMin' not in params and 'min' in params: - params['yMin'] = params['min'] - if 'yMax' not in params and 'max' in params: - params['yMax'] = params['max'] - if 'lineWidth' not in params and 'thickness' in params: - params['lineWidth'] = params['thickness'] - if 'yAxisSide' not in params: - params['yAxisSide'] = 'left' - if 'yUnitSystem' not in params: - params['yUnitSystem'] = 'si' + self.area["xmin"] = 0 + self.area["xmax"] = self.width + self.area["ymin"] = 0 + self.area["ymax"] = self.height + if "yMin" not in params and "min" in params: + params["yMin"] = params["min"] + if "yMax" not in params and "max" in params: + params["yMax"] = params["max"] + if "lineWidth" not in params and "thickness" in params: + params["lineWidth"] = params["thickness"] + if "yAxisSide" not in params: + params["yAxisSide"] = "left" + if "yUnitSystem" not in params: + params["yUnitSystem"] = "si" else: - params['yUnitSystem'] = force_text(params['yUnitSystem']).lower() - if params['yUnitSystem'] not in UnitSystems: - params['yUnitSystem'] = 'si' + params["yUnitSystem"] = force_text(params["yUnitSystem"]).lower() + if params["yUnitSystem"] not in UnitSystems: + params["yUnitSystem"] = "si" self.params = params # Don't do any of the special right y-axis stuff if we're drawing 2 # y-axes. if self.secondYAxis: - params['yAxisSide'] = 'left' + params["yAxisSide"] = "left" # When Y Axis is labeled on the right, we subtract x-axis positions # from the max, instead of adding to the minimum - if self.params.get('yAxisSide') == 'right': + if self.params.get("yAxisSide") == "right": self.margin = self.width # Now to setup our LineGraph specific options - self.lineWidth = float(params.get('lineWidth', 1.2)) - self.lineMode = params.get('lineMode', 'slope').lower() + self.lineWidth = float(params.get("lineWidth", 1.2)) + self.lineMode = params.get("lineMode", "slope").lower() self.connectedLimit = params.get("connectedLimit", INFINITY) assert self.lineMode in self.validLineModes, "Invalid line mode!" - self.areaMode = params.get('areaMode', 'none').lower() + self.areaMode = params.get("areaMode", "none").lower() assert self.areaMode in self.validAreaModes, "Invalid area mode!" - self.pieMode = params.get('pieMode', 'maximum').lower() + self.pieMode = params.get("pieMode", "maximum").lower() assert self.pieMode in self.validPieModes, "Invalid pie mode!" # Line mode slope does not work (or even make sense) for series that # have only one datapoint. So if any series have one datapoint we # force staircase mode. - if self.lineMode == 'slope': + if self.lineMode == "slope": for series in self.data: if len(series) == 1: - self.lineMode = 'staircase' + self.lineMode = "staircase" break if self.secondYAxis: for series in self.data: - if 'secondYAxis' in series.options: - if 'rightWidth' in params: - series.options['lineWidth'] = params['rightWidth'] - if 'rightDashed' in params: - series.options['dashed'] = params['rightDashed'] - if 'rightColor' in params: - series.color = params['rightColor'] + if "secondYAxis" in series.options: + if "rightWidth" in params: + series.options["lineWidth"] = params["rightWidth"] + if "rightDashed" in params: + series.options["dashed"] = params["rightDashed"] + if "rightColor" in params: + series.color = params["rightColor"] else: - if 'leftWidth' in params: - series.options['lineWidth'] = params['leftWidth'] - if 'leftDashed' in params: - series.options['dashed'] = params['leftDashed'] - if 'leftColor' in params: - series.color = params['leftColor'] + if "leftWidth" in params: + series.options["lineWidth"] = params["leftWidth"] + if "leftDashed" in params: + series.options["dashed"] = params["leftDashed"] + if "leftColor" in params: + series.color = params["leftColor"] for series in self.data: - if not hasattr(series, 'color'): + if not hasattr(series, "color"): series.color = next(self.colors) - titleSize = self.defaultFontParams['size'] + math.floor( - math.log(self.defaultFontParams['size'])) + titleSize = self.defaultFontParams["size"] + math.floor( + math.log(self.defaultFontParams["size"]) + ) self.setFont(size=titleSize) self.setColor(self.foregroundColor) - if params.get('title'): - self.drawTitle(force_text(params['title'])) - if params.get('vtitle'): - self.drawVTitle(force_text(params['vtitle'])) - if self.secondYAxis and params.get('vtitleRight'): - self.drawVTitle(force_text(params['vtitleRight']), rightAlign=True) + if params.get("title"): + self.drawTitle(force_text(params["title"])) + if params.get("vtitle"): + self.drawVTitle(force_text(params["vtitle"])) + if self.secondYAxis and params.get("vtitleRight"): + self.drawVTitle( + force_text(params["vtitleRight"]), rightAlign=True + ) self.setFont() - if not params.get('hideLegend', len(self.data) > 10): + if not params.get("hideLegend", len(self.data) > 10): elements = [] - hideNull = params.get('hideNullFromLegend', False) + hideNull = params.get("hideNullFromLegend", False) for series in self.data: if series.name: - if not(hideNull and all(v is None for v in list(series))): - elements.append(( - unquote_plus(series.name), - series.color, - series.options.get('secondYAxis'))) + if not ( + hideNull and all(v is None for v in list(series)) + ): + elements.append( + ( + unquote_plus(series.name), + series.color, + series.options.get("secondYAxis"), + ) + ) if len(elements) > 0: - self.drawLegend(elements, params.get('uniqueLegend', False)) + self.drawLegend(elements, params.get("uniqueLegend", False)) # Setup axes, labels, and grid # First we adjust the drawing area size to fit X-axis labels - if ( - not self.params.get('hideAxes', False) and - not self.params.get('hideXAxis', False) + if not self.params.get("hideAxes", False) and not self.params.get( + "hideXAxis", False ): - self.area['ymax'] -= self.getExtents()['maxAscent'] * 2 + self.area["ymax"] -= self.getExtents()["maxAscent"] * 2 self.startTime = min([series.start for series in self.data]) - if ( - self.lineMode == 'staircase' or - set([len(series) for series in self.data]) == set([2]) - ): + if self.lineMode == "staircase" or set( + [len(series) for series in self.data] + ) == set([2]): self.endTime = max([series.end for series in self.data]) else: - self.endTime = max([ - (series.end - series.step) for series in self.data]) + self.endTime = max( + [(series.end - series.step) for series in self.data] + ) self.timeRange = self.endTime - self.startTime # Now we consolidate our data points to fit in the currently estimated # drawing area self.consolidateDataPoints() - self.encodeHeader('axes') + self.encodeHeader("axes") # Now its time to fully configure the Y-axis and determine the space # required for Y-axis labels. Since we'll probably have to squeeze the @@ -1288,22 +1454,22 @@ def drawGraph(self, **params): # data points, which in turn means re-scaling the Y axis, this process # will repeat until we have accurate Y labels and enough space to fit # our data points - currentXMin = self.area['xmin'] - currentXMax = self.area['xmax'] + currentXMin = self.area["xmin"] + currentXMax = self.area["xmax"] if self.secondYAxis: self.setupTwoYAxes() else: self.setupYAxis() while ( - currentXMin != self.area['xmin'] or - currentXMax != self.area['xmax'] + currentXMin != self.area["xmin"] + or currentXMax != self.area["xmax"] ): # see if the Y-labels require more space # this can cause the Y values to change self.consolidateDataPoints() # so let's keep track of the previous Y-label space requirements - currentXMin = self.area['xmin'] - currentXMax = self.area['xmax'] + currentXMin = self.area["xmin"] + currentXMax = self.area["xmax"] if self.secondYAxis: # and recalculate their new requirements self.setupTwoYAxes() else: @@ -1313,38 +1479,40 @@ def drawGraph(self, **params): # won't affect the drawing area) self.setupXAxis() - if not self.params.get('hideAxes', False): + if not self.params.get("hideAxes", False): self.drawLabels() - if not self.params.get('hideGrid', False): + if not self.params.get("hideGrid", False): # hideAxes implies hideGrid - self.encodeHeader('grid') + self.encodeHeader("grid") self.drawGridLines() # Finally, draw the graph lines - self.encodeHeader('lines') + self.encodeHeader("lines") self.drawLines() def drawVTitle(self, text, rightAlign=False): - lineHeight = self.getExtents()['maxHeight'] + lineHeight = self.getExtents()["maxHeight"] if rightAlign: - self.encodeHeader('vtitleRight') - x = self.area['xmax'] - lineHeight + self.encodeHeader("vtitleRight") + x = self.area["xmax"] - lineHeight y = self.height / 2 - for line in text.split('\n'): - self.drawText(line, x, y, align='center', valign='baseline', - rotate=90) + for line in text.split("\n"): + self.drawText( + line, x, y, align="center", valign="baseline", rotate=90 + ) x -= lineHeight - self.area['xmax'] = x - self.margin - lineHeight + self.area["xmax"] = x - self.margin - lineHeight else: - self.encodeHeader('vtitle') - x = self.area['xmin'] + lineHeight + self.encodeHeader("vtitle") + x = self.area["xmin"] + lineHeight y = self.height / 2 - for line in text.split('\n'): - self.drawText(line, x, y, align='center', valign='baseline', - rotate=270) + for line in text.split("\n"): + self.drawText( + line, x, y, align="center", valign="baseline", rotate=270 + ) x += lineHeight - self.area['xmin'] = x + self.margin + lineHeight + self.area["xmin"] = x + self.margin + lineHeight def getYCoord(self, value, side=None): if "left" == side: @@ -1367,7 +1535,7 @@ def getYCoord(self, value, side=None): highestValue = yTop lowestValue = yBottom - pixelRange = self.area['ymax'] - self.area['ymin'] + pixelRange = self.area["ymax"] - self.area["ymin"] relativeValue = value - lowestValue valueRange = highestValue - lowestValue @@ -1375,18 +1543,20 @@ def getYCoord(self, value, side=None): if self.logBase: if value <= 0: return None - relativeValue = ( - math.log(value, self.logBase) - - math.log(lowestValue, self.logBase)) + relativeValue = math.log(value, self.logBase) - math.log( + lowestValue, self.logBase + ) valueRange = math.log(highestValue, self.logBase) - math.log( - lowestValue, self.logBase) + lowestValue, self.logBase + ) pixelToValueRatio = pixelRange / valueRange valueInPixels = pixelToValueRatio * relativeValue - return self.area['ymax'] - valueInPixels + return self.area["ymax"] - valueInPixels - def drawLines(self, width=None, dash=None, linecap='butt', - linejoin='miter'): + def drawLines( + self, width=None, dash=None, linecap="butt", linejoin="miter" + ): if not width: width = self.lineWidth self.ctx.set_line_width(width) @@ -1396,34 +1566,38 @@ def drawLines(self, width=None, dash=None, linecap='butt', self.ctx.set_dash(dash, 1) else: self.ctx.set_dash([], 0) - self.ctx.set_line_cap({ - 'butt': cairo.LINE_CAP_BUTT, - 'round': cairo.LINE_CAP_ROUND, - 'square': cairo.LINE_CAP_SQUARE, - }[linecap]) - self.ctx.set_line_join({ - 'miter': cairo.LINE_JOIN_MITER, - 'round': cairo.LINE_JOIN_ROUND, - 'bevel': cairo.LINE_JOIN_BEVEL, - }[linejoin]) + self.ctx.set_line_cap( + { + "butt": cairo.LINE_CAP_BUTT, + "round": cairo.LINE_CAP_ROUND, + "square": cairo.LINE_CAP_SQUARE, + }[linecap] + ) + self.ctx.set_line_join( + { + "miter": cairo.LINE_JOIN_MITER, + "round": cairo.LINE_JOIN_ROUND, + "bevel": cairo.LINE_JOIN_BEVEL, + }[linejoin] + ) # check whether there is an stacked metric singleStacked = False for series in self.data: - if 'stacked' in series.options: + if "stacked" in series.options: singleStacked = True if singleStacked: self.data = sort_stacked(self.data) # stack the values - if self.areaMode == 'stacked' and not self.secondYAxis: + if self.areaMode == "stacked" and not self.secondYAxis: # TODO Allow stacked area mode with secondYAxis total = [] for series in self.data: - if 'drawAsInfinite' in series.options: + if "drawAsInfinite" in series.options: continue - series.options['stacked'] = True + series.options["stacked"] = True for i in range(len(series)): if len(total) <= i: total.append(0) @@ -1432,41 +1606,47 @@ def drawLines(self, width=None, dash=None, linecap='butt', original = series[i] series[i] += total[i] total[i] += original - elif self.areaMode == 'first': - self.data[0].options['stacked'] = True - elif self.areaMode == 'all': + elif self.areaMode == "first": + self.data[0].options["stacked"] = True + elif self.areaMode == "all": for series in self.data: - if 'drawAsInfinite' not in series.options: - series.options['stacked'] = True + if "drawAsInfinite" not in series.options: + series.options["stacked"] = True # apply alpha channel and create separate stroke series - if self.params.get('areaAlpha'): + if self.params.get("areaAlpha"): try: - alpha = float(self.params['areaAlpha']) + alpha = float(self.params["areaAlpha"]) except ValueError: alpha = 0.5 strokeSeries = [] for series in self.data: - if 'stacked' in series.options: - series.options['alpha'] = alpha + if "stacked" in series.options: + series.options["alpha"] = alpha newSeries = TimeSeries( - series.name, series.start, series.end, + series.name, + series.start, + series.end, series.step * series.valuesPerPoint, - [x for x in series]) + [x for x in series], + ) newSeries.xStep = series.xStep newSeries.color = series.color - if 'secondYAxis' in series.options: - newSeries.options['secondYAxis'] = True + if "secondYAxis" in series.options: + newSeries.options["secondYAxis"] = True strokeSeries.append(newSeries) self.data += strokeSeries # setup the clip region self.ctx.set_line_width(1.0) - self.ctx.rectangle(self.area['xmin'], self.area['ymin'], - self.area['xmax'] - self.area['xmin'], - self.area['ymax'] - self.area['ymin']) + self.ctx.rectangle( + self.area["xmin"], + self.area["ymin"], + self.area["xmax"] - self.area["xmin"], + self.area["ymax"] - self.area["ymin"], + ) self.ctx.clip() self.ctx.set_line_width(originalWidth) @@ -1476,7 +1656,7 @@ def drawLines(self, width=None, dash=None, linecap='butt', for series in self.data: - if 'stacked' not in series.options: + if "stacked" not in series.options: # stacked areas are always drawn first. if this series is not # stacked, we finished stacking. reset the clip region so # lines can show up on top of the stacked areas. @@ -1484,28 +1664,31 @@ def drawLines(self, width=None, dash=None, linecap='butt', clipRestored = True self.ctx.restore() - if 'lineWidth' in series.options: - self.ctx.set_line_width(series.options['lineWidth']) + if "lineWidth" in series.options: + self.ctx.set_line_width(series.options["lineWidth"]) - if 'dashed' in series.options: - self.ctx.set_dash([series.options['dashed']], 1) + if "dashed" in series.options: + self.ctx.set_dash([series.options["dashed"]], 1) else: self.ctx.set_dash([], 0) # Shift the beginning of drawing area to the start of the series # if the graph itself has a larger range missingPoints = (series.start - self.startTime) / series.step - startShift = series.xStep * (missingPoints / series.valuesPerPoint) - x = float(self.area['xmin']) + startShift + (self.lineWidth / 2.0) - y = float(self.area['ymin']) + startShift = series.xStep * ( + missingPoints / series.valuesPerPoint + ) + x = float(self.area["xmin"]) + startShift + (self.lineWidth / 2.0) + y = float(self.area["ymin"]) startX = x - if series.options.get('invisible'): + if series.options.get("invisible"): self.setColor(series.color, 0, True) else: - self.setColor(series.color, - series.options.get('alpha') or 1.0) + self.setColor( + series.color, series.options.get("alpha") or 1.0 + ) # The number of preceding datapoints that had a None value. consecutiveNones = 0 @@ -1514,33 +1697,40 @@ def drawLines(self, width=None, dash=None, linecap='butt', if value != value: # convert NaN to None value = None - if value is None and self.params.get('drawNullAsZero'): + if value is None and self.params.get("drawNullAsZero"): value = 0.0 if value is None: if consecutiveNones == 0: self.ctx.line_to(x, y) - if 'stacked' in series.options: + if "stacked" in series.options: # Close off and fill area before unknown interval if self.secondYAxis: - if 'secondYAxis' in series.options: + if "secondYAxis" in series.options: self.fillAreaAndClip( - x, y, startX, - self.getYCoord(0, "right")) + x, + y, + startX, + self.getYCoord(0, "right"), + ) else: self.fillAreaAndClip( - x, y, startX, - self.getYCoord(0, "left")) + x, + y, + startX, + self.getYCoord(0, "left"), + ) else: - self.fillAreaAndClip(x, y, startX, - self.getYCoord(0)) + self.fillAreaAndClip( + x, y, startX, self.getYCoord(0) + ) x += series.xStep consecutiveNones += 1 else: if self.secondYAxis: - if 'secondYAxis' in series.options: + if "secondYAxis" in series.options: y = self.getYCoord(value, "right") else: y = self.getYCoord(value, "left") @@ -1552,9 +1742,9 @@ def drawLines(self, width=None, dash=None, linecap='butt', elif y < 0: y = 0 - if 'drawAsInfinite' in series.options and value > 0: - self.ctx.move_to(x, self.area['ymax']) - self.ctx.line_to(x, self.area['ymin']) + if "drawAsInfinite" in series.options and value > 0: + self.ctx.move_to(x, self.area["ymax"]) + self.ctx.line_to(x, self.area["ymin"]) self.ctx.stroke() x += series.xStep continue @@ -1562,7 +1752,7 @@ def drawLines(self, width=None, dash=None, linecap='butt', if consecutiveNones > 0: startX = x - if self.lineMode == 'staircase': + if self.lineMode == "staircase": if consecutiveNones > 0: self.ctx.move_to(x, y) else: @@ -1571,20 +1761,20 @@ def drawLines(self, width=None, dash=None, linecap='butt', x += series.xStep self.ctx.line_to(x, y) - elif self.lineMode == 'slope': + elif self.lineMode == "slope": if consecutiveNones > 0: self.ctx.move_to(x, y) self.ctx.line_to(x, y) x += series.xStep - elif self.lineMode == 'connected': + elif self.lineMode == "connected": # If if the gap is larger than the connectedLimit or # if this is the first non-None datapoint in the # series, start drawing from that datapoint. if ( - consecutiveNones > self.connectedLimit or - consecutiveNones == index + consecutiveNones > self.connectedLimit + or consecutiveNones == index ): self.ctx.move_to(x, y) @@ -1593,13 +1783,13 @@ def drawLines(self, width=None, dash=None, linecap='butt', consecutiveNones = 0 - if 'stacked' in series.options: - if self.lineMode == 'staircase': + if "stacked" in series.options: + if self.lineMode == "staircase": xPos = x else: - xPos = x-series.xStep + xPos = x - series.xStep if self.secondYAxis: - if 'secondYAxis' in series.options: + if "secondYAxis" in series.options: areaYFrom = self.getYCoord(0, "right") else: areaYFrom = self.getYCoord(0, "left") @@ -1613,7 +1803,7 @@ def drawLines(self, width=None, dash=None, linecap='butt', # return to the original line width self.ctx.set_line_width(originalWidth) - if 'dashed' in series.options: + if "dashed" in series.options: # if we changed the dash setting before, change it back now if dash: self.ctx.set_dash(dash, 1) @@ -1621,15 +1811,15 @@ def drawLines(self, width=None, dash=None, linecap='butt', self.ctx.set_dash([], 0) def fillAreaAndClip(self, x, y, startX=None, areaYFrom=None): - startX = (startX or self.area['xmin']) - areaYFrom = (areaYFrom or self.area['ymax']) + startX = startX or self.area["xmin"] + areaYFrom = areaYFrom or self.area["ymax"] pattern = self.ctx.copy_path() # fill self.ctx.line_to(x, areaYFrom) # bottom endX self.ctx.line_to(startX, areaYFrom) # bottom startX self.ctx.close_path() - if self.areaMode == 'all': + if self.areaMode == "all": self.ctx.fill_preserve() else: self.ctx.fill() @@ -1637,64 +1827,76 @@ def fillAreaAndClip(self, x, y, startX=None, areaYFrom=None): # clip above y axis self.ctx.append_path(pattern) self.ctx.line_to(x, areaYFrom) # yZero endX - self.ctx.line_to(self.area['xmax'], areaYFrom) # yZero right - self.ctx.line_to(self.area['xmax'], self.area['ymin']) # top right - self.ctx.line_to(self.area['xmin'], self.area['ymin']) # top left - self.ctx.line_to(self.area['xmin'], areaYFrom) # yZero left + self.ctx.line_to(self.area["xmax"], areaYFrom) # yZero right + self.ctx.line_to(self.area["xmax"], self.area["ymin"]) # top right + self.ctx.line_to(self.area["xmin"], self.area["ymin"]) # top left + self.ctx.line_to(self.area["xmin"], areaYFrom) # yZero left self.ctx.line_to(startX, areaYFrom) # yZero startX # clip below y axis self.ctx.line_to(x, areaYFrom) # yZero endX - self.ctx.line_to(self.area['xmax'], areaYFrom) # yZero right - self.ctx.line_to(self.area['xmax'], self.area['ymax']) # bottom right - self.ctx.line_to(self.area['xmin'], self.area['ymax']) # bottom left - self.ctx.line_to(self.area['xmin'], areaYFrom) # yZero left + self.ctx.line_to(self.area["xmax"], areaYFrom) # yZero right + self.ctx.line_to(self.area["xmax"], self.area["ymax"]) # bottom right + self.ctx.line_to(self.area["xmin"], self.area["ymax"]) # bottom left + self.ctx.line_to(self.area["xmin"], areaYFrom) # yZero left self.ctx.line_to(startX, areaYFrom) # yZero startX self.ctx.close_path() self.ctx.clip() def consolidateDataPoints(self): numberOfPixels = self.graphWidth = ( - self.area['xmax'] - self.area['xmin'] - (self.lineWidth + 1)) + self.area["xmax"] - self.area["xmin"] - (self.lineWidth + 1) + ) for series in self.data: numberOfDataPoints = self.timeRange / series.step - minXStep = float(self.params.get('minXStep', 1.0)) + minXStep = float(self.params.get("minXStep", 1.0)) divisor = self.timeRange / series.step or 1 bestXStep = numberOfPixels / divisor if bestXStep < minXStep: drawableDataPoints = int(numberOfPixels / minXStep) pointsPerPixel = math.ceil( - float(numberOfDataPoints) / float(drawableDataPoints)) + float(numberOfDataPoints) / float(drawableDataPoints) + ) series.consolidate(pointsPerPixel) series.xStep = ( - numberOfPixels * pointsPerPixel) / numberOfDataPoints + numberOfPixels * pointsPerPixel + ) / numberOfDataPoints else: series.xStep = bestXStep def setupYAxis(self): - drawNullAsZero = self.params.get('drawNullAsZero') - stacked = (self.areaMode == 'stacked') + drawNullAsZero = self.params.get("drawNullAsZero") + stacked = self.areaMode == "stacked" - (yMinValue, yMaxValue) = dataLimits(self.data, drawNullAsZero, - stacked) + (yMinValue, yMaxValue) = dataLimits( + self.data, drawNullAsZero, stacked + ) if self.logBase: - yTics = _LogAxisTics(yMinValue, yMaxValue, - unitSystem=self.params.get('yUnitSystem'), - base=self.logBase) + yTics = _LogAxisTics( + yMinValue, + yMaxValue, + unitSystem=self.params.get("yUnitSystem"), + base=self.logBase, + ) else: - yTics = _LinearAxisTics(yMinValue, yMaxValue, - unitSystem=self.params.get('yUnitSystem')) - - yTics.applySettings(axisMin=self.params.get('yMin'), - axisMax=self.params.get('yMax'), - axisLimit=self.params.get('yLimit')) - - if 'yStep' in self.params: - yTics.setStep(self.params['yStep']) + yTics = _LinearAxisTics( + yMinValue, + yMaxValue, + unitSystem=self.params.get("yUnitSystem"), + ) + + yTics.applySettings( + axisMin=self.params.get("yMin"), + axisMax=self.params.get("yMax"), + axisLimit=self.params.get("yLimit"), + ) + + if "yStep" in self.params: + yTics.setStep(self.params["yStep"]) else: - yDivisors = str(self.params.get('yDivisors', '4,5,6')) - yDivisors = [int(d) for d in yDivisors.split(',')] - binary = self.params.get('yUnitSystem') == 'binary' + yDivisors = str(self.params.get("yDivisors", "4,5,6")) + yDivisors = [int(d) for d in yDivisors.split(",")] + binary = self.params.get("yUnitSystem") == "binary" yTics.chooseStep(divisors=yDivisors, binary=binary) yTics.chooseLimits() @@ -1705,74 +1907,94 @@ def setupYAxis(self): self.yTop = yTics.top self.ySpan = yTics.span - if not self.params.get('hideAxes', False): + if not self.params.get("hideAxes", False): # Create and measure the Y-labels self.yLabelValues = yTics.getLabelValues() - self.yLabels = [yTics.makeLabel(value) - for value in self.yLabelValues] - self.yLabelWidth = max([ - self.getExtents(label)['width'] for label in self.yLabels]) - - if not self.params.get('hideYAxis'): - if self.params.get('yAxisSide') == 'left': + self.yLabels = [ + yTics.makeLabel(value) for value in self.yLabelValues + ] + self.yLabelWidth = max( + [self.getExtents(label)["width"] for label in self.yLabels] + ) + + if not self.params.get("hideYAxis"): + if self.params.get("yAxisSide") == "left": # Scoot the graph over to the left just enough to fit the # y-labels: xMin = self.margin + (self.yLabelWidth * 1.02) - if self.area['xmin'] < xMin: - self.area['xmin'] = xMin + if self.area["xmin"] < xMin: + self.area["xmin"] = xMin else: # Scoot the graph over to the right just enough to fit # # the y-labels: xMin = 0 xMax = self.margin - (self.yLabelWidth * 1.02) - if self.area['xmax'] >= xMax: - self.area['xmax'] = xMax + if self.area["xmax"] >= xMax: + self.area["xmax"] = xMax else: self.yLabelValues = [] self.yLabels = [] self.yLabelWidth = 0.0 def setupTwoYAxes(self): - drawNullAsZero = self.params.get('drawNullAsZero') - stacked = (self.areaMode == 'stacked') + drawNullAsZero = self.params.get("drawNullAsZero") + stacked = self.areaMode == "stacked" - (yMinValueL, yMaxValueL) = dataLimits(self.dataLeft, drawNullAsZero, - stacked) - (yMinValueR, yMaxValueR) = dataLimits(self.dataRight, drawNullAsZero, - stacked) + (yMinValueL, yMaxValueL) = dataLimits( + self.dataLeft, drawNullAsZero, stacked + ) + (yMinValueR, yMaxValueR) = dataLimits( + self.dataRight, drawNullAsZero, stacked + ) # TODO: Allow separate bases for L & R Axes. if self.logBase: - yTicsL = _LogAxisTics(yMinValueL, yMaxValueL, - unitSystem=self.params.get('yUnitSystem'), - base=self.logBase) - yTicsR = _LogAxisTics(yMinValueR, yMaxValueR, - unitSystem=self.params.get('yUnitSystem'), - base=self.logBase) + yTicsL = _LogAxisTics( + yMinValueL, + yMaxValueL, + unitSystem=self.params.get("yUnitSystem"), + base=self.logBase, + ) + yTicsR = _LogAxisTics( + yMinValueR, + yMaxValueR, + unitSystem=self.params.get("yUnitSystem"), + base=self.logBase, + ) else: - yTicsL = _LinearAxisTics(yMinValueL, yMaxValueL, - unitSystem=self.params.get('yUnitSystem')) - yTicsR = _LinearAxisTics(yMinValueR, yMaxValueR, - unitSystem=self.params.get('yUnitSystem')) - - yTicsL.applySettings(axisMin=self.params.get('yMinLeft'), - axisMax=self.params.get('yMaxLeft'), - axisLimit=self.params.get('yLimitLeft')) - yTicsR.applySettings(axisMin=self.params.get('yMinRight'), - axisMax=self.params.get('yMaxRight'), - axisLimit=self.params.get('yLimitRight')) - - yDivisors = str(self.params.get('yDivisors', '4,5,6')) - yDivisors = [int(d) for d in yDivisors.split(',')] - binary = self.params.get('yUnitSystem') == 'binary' - - if 'yStepLeft' in self.params: - yTicsL.setStep(self.params['yStepLeft']) + yTicsL = _LinearAxisTics( + yMinValueL, + yMaxValueL, + unitSystem=self.params.get("yUnitSystem"), + ) + yTicsR = _LinearAxisTics( + yMinValueR, + yMaxValueR, + unitSystem=self.params.get("yUnitSystem"), + ) + + yTicsL.applySettings( + axisMin=self.params.get("yMinLeft"), + axisMax=self.params.get("yMaxLeft"), + axisLimit=self.params.get("yLimitLeft"), + ) + yTicsR.applySettings( + axisMin=self.params.get("yMinRight"), + axisMax=self.params.get("yMaxRight"), + axisLimit=self.params.get("yLimitRight"), + ) + + yDivisors = str(self.params.get("yDivisors", "4,5,6")) + yDivisors = [int(d) for d in yDivisors.split(",")] + binary = self.params.get("yUnitSystem") == "binary" + + if "yStepLeft" in self.params: + yTicsL.setStep(self.params["yStepLeft"]) else: yTicsL.chooseStep(divisors=yDivisors, binary=binary) - if 'yStepRight' in self.params: - yTicsR.setStep(self.params['yStepRight']) + if "yStepRight" in self.params: + yTicsR.setStep(self.params["yStepRight"]) else: yTicsR.chooseStep(divisors=yDivisors, binary=binary) @@ -1794,32 +2016,37 @@ def setupTwoYAxes(self): self.yLabelValuesL = yTicsL.getLabelValues() self.yLabelValuesR = yTicsR.getLabelValues() - self.yLabelsL = [yTicsL.makeLabel(value) - for value in self.yLabelValuesL] - self.yLabelsR = [yTicsR.makeLabel(value) - for value in self.yLabelValuesR] - - self.yLabelWidthL = max([ - self.getExtents(label)['width'] for label in self.yLabelsL]) - self.yLabelWidthR = max([ - self.getExtents(label)['width'] for label in self.yLabelsR]) + self.yLabelsL = [ + yTicsL.makeLabel(value) for value in self.yLabelValuesL + ] + self.yLabelsR = [ + yTicsR.makeLabel(value) for value in self.yLabelValuesR + ] + + self.yLabelWidthL = max( + [self.getExtents(label)["width"] for label in self.yLabelsL] + ) + self.yLabelWidthR = max( + [self.getExtents(label)["width"] for label in self.yLabelsR] + ) # scoot the graph over to the left just enough to fit the y-labels # xMin = self.margin + self.margin + (self.yLabelWidthL * 1.02) xMin = self.margin + (self.yLabelWidthL * 1.02) - if self.area['xmin'] < xMin: - self.area['xmin'] = xMin + if self.area["xmin"] < xMin: + self.area["xmin"] = xMin # scoot the graph over to the right just enough to fit the y-labels xMax = self.width - (self.yLabelWidthR * 1.02) - if self.area['xmax'] >= xMax: - self.area['xmax'] = xMax + if self.area["xmax"] >= xMax: + self.area["xmax"] = xMax def setupXAxis(self): from ..app import app + if self.userTimeZone: tzinfo = pytz.timezone(self.userTimeZone) else: - tzinfo = pytz.timezone(app.config['TIME_ZONE']) + tzinfo = pytz.timezone(app.config["TIME_ZONE"]) self.start_dt = datetime.fromtimestamp(self.startTime, tzinfo) self.end_dt = datetime.fromtimestamp(self.endTime, tzinfo) @@ -1829,29 +2056,34 @@ def setupXAxis(self): self.xScaleFactor = float(self.graphWidth) / float(self.timeRange) potential = [ - c for c in xAxisConfigs if c['seconds'] <= secondsPerPixel and - c.get('maxInterval', self.timeRange + 1) >= self.timeRange] + c + for c in xAxisConfigs + if c["seconds"] <= secondsPerPixel + and c.get("maxInterval", self.timeRange + 1) >= self.timeRange + ] if potential: self.xConf = potential[-1] else: self.xConf = xAxisConfigs[-1] - self.xLabelStep = self.xConf['labelUnit'] * self.xConf['labelStep'] + self.xLabelStep = self.xConf["labelUnit"] * self.xConf["labelStep"] self.xMinorGridStep = ( - self.xConf['minorGridUnit'] * self.xConf['minorGridStep']) + self.xConf["minorGridUnit"] * self.xConf["minorGridStep"] + ) self.xMajorGridStep = ( - self.xConf['majorGridUnit'] * self.xConf['majorGridStep']) + self.xConf["majorGridUnit"] * self.xConf["majorGridStep"] + ) def drawLabels(self): # Draw the Y-labels - if not self.params.get('hideYAxis'): + if not self.params.get("hideYAxis"): if not self.secondYAxis: for value, label in zip(self.yLabelValues, self.yLabels): - if self.params.get('yAxisSide') == 'left': - x = self.area['xmin'] - (self.yLabelWidth * 0.02) + if self.params.get("yAxisSide") == "left": + x = self.area["xmin"] - (self.yLabelWidth * 0.02) else: # Inverted for right side Y Axis - x = self.area['xmax'] + (self.yLabelWidth * 0.02) + x = self.area["xmax"] + (self.yLabelWidth * 0.02) y = self.getYCoord(value) if y is None: @@ -1859,50 +2091,57 @@ def drawLabels(self): elif y < 0: y = 0 - if self.params.get('yAxisSide') == 'left': - self.drawText(label, x, y, align='right', - valign='middle') + if self.params.get("yAxisSide") == "left": + self.drawText( + label, x, y, align="right", valign="middle" + ) else: # Inverted for right side Y Axis - self.drawText(label, x, y, align='left', - valign='middle') + self.drawText( + label, x, y, align="left", valign="middle" + ) else: # Draws a right side and a Left side axis for valueL, labelL in zip(self.yLabelValuesL, self.yLabelsL): - xL = self.area['xmin'] - (self.yLabelWidthL * 0.02) + xL = self.area["xmin"] - (self.yLabelWidthL * 0.02) yL = self.getYCoord(valueL, "left") if yL is None: value = None elif yL < 0: yL = 0 - self.drawText(labelL, xL, yL, align='right', - valign='middle') + self.drawText( + labelL, xL, yL, align="right", valign="middle" + ) # Right Side for valueR, labelR in zip(self.yLabelValuesR, self.yLabelsR): # Inverted for right side Y Axis - xR = self.area['xmax'] + (self.yLabelWidthR * 0.02) + 3 + xR = self.area["xmax"] + (self.yLabelWidthR * 0.02) + 3 yR = self.getYCoord(valueR, "right") if yR is None: valueR = None elif yR < 0: yR = 0 # Inverted for right side Y Axis - self.drawText(labelR, xR, yR, align='left', - valign='middle') + self.drawText( + labelR, xR, yR, align="left", valign="middle" + ) - if not self.params.get('hideXAxis'): - dt, x_label_delta = find_x_times(self.start_dt, - self.xConf['labelUnit'], - self.xConf['labelStep']) + if not self.params.get("hideXAxis"): + dt, x_label_delta = find_x_times( + self.start_dt, + self.xConf["labelUnit"], + self.xConf["labelStep"], + ) # Draw the X-labels - xFormat = self.params.get('xFormat', self.xConf['format']) + xFormat = self.params.get("xFormat", self.xConf["format"]) while dt < self.end_dt: label = dt.strftime(xFormat) - x = self.area['xmin'] + ( - to_seconds(dt - self.start_dt) * self.xScaleFactor) - y = self.area['ymax'] + self.getExtents()['maxAscent'] - self.drawText(label, x, y, align='center', valign='top') + x = self.area["xmin"] + ( + to_seconds(dt - self.start_dt) * self.xScaleFactor + ) + y = self.area["ymax"] + self.getExtents()["maxAscent"] + self.drawText(label, x, y, align="center", valign="top") dt += x_label_delta def drawGridLines(self): @@ -1910,8 +2149,8 @@ def drawGridLines(self): # Just using the left side info for the grid. # Horizontal grid lines - leftSide = self.area['xmin'] - rightSide = self.area['xmax'] + leftSide = self.area["xmin"] + rightSide = self.area["xmax"] labels = [] if self.secondYAxis: labels = self.yLabelValuesL @@ -1920,8 +2159,11 @@ def drawGridLines(self): for i, value in enumerate(labels): self.ctx.set_line_width(0.4) - self.setColor(self.params.get('majorGridLineColor', - self.defaultMajorGridLineColor)) + self.setColor( + self.params.get( + "majorGridLineColor", self.defaultMajorGridLineColor + ) + ) if self.secondYAxis: y = self.getYCoord(value, "left") @@ -1937,12 +2179,12 @@ def drawGridLines(self): # draw minor gridlines if this isn't the last label if self.minorY >= 1 and i < (len(labels) - 1): # in case graphite supports inverted Y axis now or someday - valueLower, valueUpper = sorted((value, labels[i+1])) + valueLower, valueUpper = sorted((value, labels[i + 1])) # each minor gridline is 1/minorY apart from the nearby # gridlines. we calculate that distance, for adding to the # value in the loop. - distance = ((valueUpper - valueLower) / float(1 + self.minorY)) + distance = (valueUpper - valueLower) / float(1 + self.minorY) # starting from the initial valueLower, we add the minor # distance for each minor gridline that we wish to draw, and @@ -1950,12 +2192,15 @@ def drawGridLines(self): for minor in range(self.minorY): self.ctx.set_line_width(0.3) self.setColor( - self.params.get('minorGridLineColor', - self.defaultMinorGridLineColor)) + self.params.get( + "minorGridLineColor", + self.defaultMinorGridLineColor, + ) + ) # the current minor gridline value is halfway between the # current and next major gridline values - value = valueLower + ((1+minor) * distance) + value = valueLower + ((1 + minor) * distance) if self.logBase: yTopFactor = self.logBase * self.logBase @@ -1981,22 +2226,28 @@ def drawGridLines(self): self.ctx.stroke() # Vertical grid lines - top = self.area['ymin'] - bottom = self.area['ymax'] + top = self.area["ymin"] + bottom = self.area["ymax"] # First we do the minor grid lines (majors will paint over them) self.ctx.set_line_width(0.25) - self.setColor(self.params.get('minorGridLineColor', - self.defaultMinorGridLineColor)) + self.setColor( + self.params.get( + "minorGridLineColor", self.defaultMinorGridLineColor + ) + ) dt, x_minor_delta = find_x_times( - self.start_dt, self.xConf['minorGridUnit'], - self.xConf['minorGridStep']) + self.start_dt, + self.xConf["minorGridUnit"], + self.xConf["minorGridStep"], + ) while dt < self.end_dt: - x = self.area['xmin'] + ( - to_seconds(dt - self.start_dt) * self.xScaleFactor) + x = self.area["xmin"] + ( + to_seconds(dt - self.start_dt) * self.xScaleFactor + ) - if x < self.area['xmax']: + if x < self.area["xmax"]: self.ctx.move_to(x, bottom) self.ctx.line_to(x, top) self.ctx.stroke() @@ -2005,17 +2256,23 @@ def drawGridLines(self): # Now we do the major grid lines self.ctx.set_line_width(0.33) - self.setColor(self.params.get('majorGridLineColor', - self.defaultMajorGridLineColor)) - dt, x_major_delta = find_x_times(self.start_dt, - self.xConf['majorGridUnit'], - self.xConf['majorGridStep']) + self.setColor( + self.params.get( + "majorGridLineColor", self.defaultMajorGridLineColor + ) + ) + dt, x_major_delta = find_x_times( + self.start_dt, + self.xConf["majorGridUnit"], + self.xConf["majorGridStep"], + ) while dt < self.end_dt: - x = self.area['xmin'] + ( - to_seconds(dt - self.start_dt) * self.xScaleFactor) + x = self.area["xmin"] + ( + to_seconds(dt - self.start_dt) * self.xScaleFactor + ) - if x < self.area['xmax']: + if x < self.area["xmax"]: self.ctx.move_to(x, bottom) self.ctx.line_to(x, top) self.ctx.stroke() @@ -2024,27 +2281,32 @@ def drawGridLines(self): # Draw side borders for our graph area self.ctx.set_line_width(0.5) - self.ctx.move_to(self.area['xmax'], bottom) - self.ctx.line_to(self.area['xmax'], top) - self.ctx.move_to(self.area['xmin'], bottom) - self.ctx.line_to(self.area['xmin'], top) + self.ctx.move_to(self.area["xmax"], bottom) + self.ctx.line_to(self.area["xmax"], top) + self.ctx.move_to(self.area["xmin"], bottom) + self.ctx.line_to(self.area["xmin"], top) self.ctx.stroke() class PieGraph(Graph): customizable = Graph.customizable + ( - 'title', 'valueLabels', 'valueLabelsMin', 'hideLegend', 'pieLabels', - 'areaAlpha', 'valueLabelsColor', + "title", + "valueLabels", + "valueLabelsMin", + "hideLegend", + "pieLabels", + "areaAlpha", + "valueLabelsColor", ) - validValueLabels = ('none', 'number', 'percent') + validValueLabels = ("none", "number", "percent") def drawGraph(self, **params): - self.pieLabels = params.get('pieLabels', 'horizontal') + self.pieLabels = params.get("pieLabels", "horizontal") self.total = sum([t[1] for t in self.data]) - if self.params.get('areaAlpha'): + if self.params.get("areaAlpha"): try: - self.alpha = float(self.params['areaAlpha']) + self.alpha = float(self.params["areaAlpha"]) except ValueError: self.alpha = 1.0 else: @@ -2052,95 +2314,102 @@ def drawGraph(self, **params): self.slices = [] for name, value in self.data: - self.slices.append({ - 'name': name, - 'value': value, - 'percent': value / self.total, - 'color': next(self.colors), - 'alpha': self.alpha, - }) - - titleSize = self.defaultFontParams['size'] + math.floor( - math.log(self.defaultFontParams['size'])) + self.slices.append( + { + "name": name, + "value": value, + "percent": value / self.total, + "color": next(self.colors), + "alpha": self.alpha, + } + ) + + titleSize = self.defaultFontParams["size"] + math.floor( + math.log(self.defaultFontParams["size"]) + ) self.setFont(size=titleSize) self.setColor(self.foregroundColor) - if params.get('title'): - self.drawTitle(params['title']) + if params.get("title"): + self.drawTitle(params["title"]) self.setFont() - if not params.get('hideLegend', False): + if not params.get("hideLegend", False): elements = [ - (slice['name'], slice['color'], None) for slice in self.slices] + (slice["name"], slice["color"], None) for slice in self.slices + ] self.drawLegend(elements) self.drawSlices() - if params.get('valueLabelsColor'): - self.valueLabelsColor = params.get('valueLabelsColor') + if params.get("valueLabelsColor"): + self.valueLabelsColor = params.get("valueLabelsColor") else: - self.valueLabelsColor = 'black' + self.valueLabelsColor = "black" - self.valueLabelsMin = float(params.get('valueLabelsMin', 5)) - self.valueLabels = params.get('valueLabels', 'percent') + self.valueLabelsMin = float(params.get("valueLabelsMin", 5)) + self.valueLabels = params.get("valueLabels", "percent") assert self.valueLabels in self.validValueLabels, ( - "valueLabels=%s must be one of %s" % ( - self.valueLabels, self.validValueLabels)) - if self.valueLabels != 'none': + "valueLabels=%s must be one of %s" + % (self.valueLabels, self.validValueLabels) + ) + if self.valueLabels != "none": self.drawLabels() def drawSlices(self): theta = 3.0 * math.pi / 2.0 - halfX = (self.area['xmax'] - self.area['xmin']) / 2.0 - halfY = (self.area['ymax'] - self.area['ymin']) / 2.0 - self.x0 = x0 = self.area['xmin'] + halfX - self.y0 = y0 = self.area['ymin'] + halfY + halfX = (self.area["xmax"] - self.area["xmin"]) / 2.0 + halfY = (self.area["ymax"] - self.area["ymin"]) / 2.0 + self.x0 = x0 = self.area["xmin"] + halfX + self.y0 = y0 = self.area["ymin"] + halfY self.radius = radius = min(halfX, halfY) * 0.95 for slice in self.slices: - self.setColor(slice['color'], slice['alpha']) + self.setColor(slice["color"], slice["alpha"]) self.ctx.move_to(x0, y0) - phi = theta + (2 * math.pi) * slice['percent'] + phi = theta + (2 * math.pi) * slice["percent"] self.ctx.arc(x0, y0, radius, theta, phi) self.ctx.line_to(x0, y0) self.ctx.fill() - slice['midAngle'] = (theta + phi) / 2.0 - slice['midAngle'] %= 2.0 * math.pi + slice["midAngle"] = (theta + phi) / 2.0 + slice["midAngle"] %= 2.0 * math.pi theta = phi def drawLabels(self): self.setFont() self.setColor(self.valueLabelsColor) for slice in self.slices: - if self.valueLabels == 'percent': - if slice['percent'] * 100.0 < self.valueLabelsMin: + if self.valueLabels == "percent": + if slice["percent"] * 100.0 < self.valueLabelsMin: continue - label = "%%%.2f" % (slice['percent'] * 100.0) - elif self.valueLabels == 'number': - if slice['value'] < self.valueLabelsMin: + label = "%%%.2f" % (slice["percent"] * 100.0) + elif self.valueLabels == "number": + if slice["value"] < self.valueLabelsMin: continue - if ( - slice['value'] < 10 and - slice['value'] != int(slice['value']) + if slice["value"] < 10 and slice["value"] != int( + slice["value"] ): - label = "%.2f" % slice['value'] + label = "%.2f" % slice["value"] else: - label = force_text(int(slice['value'])) - theta = slice['midAngle'] + label = force_text(int(slice["value"])) + theta = slice["midAngle"] x = self.x0 + (self.radius / 2.0 * math.cos(theta)) y = self.y0 + (self.radius / 2.0 * math.sin(theta)) - if self.pieLabels == 'rotated': + if self.pieLabels == "rotated": if theta > (math.pi / 2.0) and theta <= (3.0 * math.pi / 2.0): theta -= math.pi - self.drawText(label, x, y, align='center', valign='middle', - rotate=math.degrees(theta)) + self.drawText( + label, + x, + y, + align="center", + valign="middle", + rotate=math.degrees(theta), + ) else: - self.drawText(label, x, y, align='center', valign='middle') + self.drawText(label, x, y, align="center", valign="middle") -GraphTypes = { - 'line': LineGraph, - 'pie': PieGraph, -} +GraphTypes = {"line": LineGraph, "pie": PieGraph} # Convience functions @@ -2149,8 +2418,11 @@ def safeArgs(args): Skip any items that are None, NaN, or infinite. """ - return (arg for arg in args - if arg is not None and not math.isnan(arg) and not math.isinf(arg)) + return ( + arg + for arg in args + if arg is not None and not math.isnan(arg) and not math.isinf(arg) + ) def safeMin(args): @@ -2175,8 +2447,9 @@ def dataLimits(data, drawNullAsZero=False, stacked=False): data is an array of TimeSeries objects. """ missingValues = any(None in series for series in data) - finiteData = [series for series in data - if not series.options.get('drawAsInfinite')] + finiteData = [ + series for series in data if not series.options.get("drawAsInfinite") + ] yMinValue = safeMin(safeMin(series) for series in finiteData) @@ -2204,8 +2477,8 @@ def dataLimits(data, drawNullAsZero=False, stacked=False): def sort_stacked(series_list): - stacked = [s for s in series_list if 'stacked' in s.options] - not_stacked = [s for s in series_list if 'stacked' not in s.options] + stacked = [s for s in series_list if "stacked" in s.options] + not_stacked = [s for s in series_list if "stacked" not in s.options] return stacked + not_stacked @@ -2226,7 +2499,7 @@ def format_units(v, step=None, system="si", units=None): http://en.wikipedia.org/wiki/Binary_prefix """ if v is None: - return 0, '' + return 0, "" for prefix, size in UnitSystems[system]: if condition(v, size, step): @@ -2242,7 +2515,7 @@ def format_units(v, step=None, system="si", units=None): if units: prefix = units else: - prefix = '' + prefix = "" return v, prefix @@ -2255,17 +2528,20 @@ def find_x_times(start_dt, unit, step): if unit == SEC: dt = start_dt.replace( - second=start_dt.second - (start_dt.second % step)) + second=start_dt.second - (start_dt.second % step) + ) x_delta = timedelta(seconds=step) elif unit == MIN: dt = start_dt.replace( - second=0, minute=start_dt.minute - (start_dt.minute % step)) + second=0, minute=start_dt.minute - (start_dt.minute % step) + ) x_delta = timedelta(minutes=step) elif unit == HOUR: dt = start_dt.replace( - second=0, minute=0, hour=start_dt.hour - (start_dt.hour % step)) + second=0, minute=0, hour=start_dt.hour - (start_dt.hour % step) + ) x_delta = timedelta(hours=step) elif unit == DAY: diff --git a/graphite_api/render/grammar.py b/graphite_api/render/grammar.py index a347f94..4f34a2f 100644 --- a/graphite_api/render/grammar.py +++ b/graphite_api/render/grammar.py @@ -1,9 +1,25 @@ from distutils.version import StrictVersion from pyparsing import ( - __version__, alphanums, alphas, CaselessKeyword, CaselessLiteral, Combine, - delimitedList, FollowedBy, Forward, Group, LineEnd, Literal, OneOrMore, - Optional, printables, quotedString, Regex, Word, ZeroOrMore, + __version__, + alphanums, + alphas, + CaselessKeyword, + CaselessLiteral, + Combine, + delimitedList, + FollowedBy, + Forward, + Group, + LineEnd, + Literal, + OneOrMore, + Optional, + printables, + quotedString, + Regex, + Word, + ZeroOrMore, ) grammar = Forward() @@ -11,113 +27,97 @@ expression = Forward() # Literals -intNumber = Regex(r'-?\d+')('integer') +intNumber = Regex(r"-?\d+")("integer") -floatNumber = Regex(r'-?\d+\.\d+')('float') +floatNumber = Regex(r"-?\d+\.\d+")("float") sciNumber = Combine( - (floatNumber | intNumber) + CaselessLiteral('e') + intNumber -)('scientific') + (floatNumber | intNumber) + CaselessLiteral("e") + intNumber +)("scientific") -aString = quotedString('string') +aString = quotedString("string") # Use lookahead to match only numbers in a list (can't remember why this # is necessary) afterNumber = FollowedBy(",") ^ FollowedBy(")") ^ FollowedBy(LineEnd()) number = Group( - (sciNumber + afterNumber) | - (floatNumber + afterNumber) | - (intNumber + afterNumber) -)('number') + (sciNumber + afterNumber) + | (floatNumber + afterNumber) + | (intNumber + afterNumber) +)("number") -boolean = Group( - CaselessKeyword("true") | - CaselessKeyword("false") -)('boolean') +boolean = Group(CaselessKeyword("true") | CaselessKeyword("false"))("boolean") -argname = Word(alphas + '_', alphanums + '_')('argname') -funcname = Word(alphas + '_', alphanums + '_')('funcname') +argname = Word(alphas + "_", alphanums + "_")("argname") +funcname = Word(alphas + "_", alphanums + "_")("funcname") # Symbols -leftParen = Literal('(').suppress() -rightParen = Literal(')').suppress() -comma = Literal(',').suppress() -equal = Literal('=').suppress() +leftParen = Literal("(").suppress() +rightParen = Literal(")").suppress() +comma = Literal(",").suppress() +equal = Literal("=").suppress() # Function calls # Symbols -leftBrace = Literal('{') -rightBrace = Literal('}') -leftParen = Literal('(').suppress() -rightParen = Literal(')').suppress() -comma = Literal(',').suppress() -equal = Literal('=').suppress() -backslash = Literal('\\').suppress() - -symbols = '''(){},=.'"\\''' -arg = Group( - boolean | - number | - aString | - expression -)('args*') -kwarg = Group(argname + equal + arg)('kwargs*') +leftBrace = Literal("{") +rightBrace = Literal("}") +leftParen = Literal("(").suppress() +rightParen = Literal(")").suppress() +comma = Literal(",").suppress() +equal = Literal("=").suppress() +backslash = Literal("\\").suppress() + +symbols = """(){},=.'"\\""" +arg = Group(boolean | number | aString | expression)("args*") +kwarg = Group(argname + equal + arg)("kwargs*") # lookahead to prevent failing on equals args = delimitedList(~kwarg + arg) kwargs = delimitedList(kwarg) call = Group( - funcname + leftParen + - Optional( - args + Optional( - comma + kwargs - ) - ) + rightParen -)('call') + funcname + + leftParen + + Optional(args + Optional(comma + kwargs)) + + rightParen +)("call") # Metric pattern (aka. pathExpression) -validMetricChars = ''.join((set(printables) - set(symbols))) +validMetricChars = "".join((set(printables) - set(symbols))) escapedChar = backslash + Word(symbols, exact=1) -partialPathElem = Combine( - OneOrMore( - escapedChar | Word(validMetricChars) - ) -) +partialPathElem = Combine(OneOrMore(escapedChar | Word(validMetricChars))) matchEnum = Combine( - leftBrace + - delimitedList(partialPathElem, combine=True) + - rightBrace + leftBrace + delimitedList(partialPathElem, combine=True) + rightBrace ) pathElement = Combine( - Group(partialPathElem | matchEnum) + - ZeroOrMore(matchEnum | partialPathElem) + Group(partialPathElem | matchEnum) + + ZeroOrMore(matchEnum | partialPathElem) +) +pathExpression = delimitedList(pathElement, delim=".", combine=True)( + "pathExpression" ) -pathExpression = delimitedList(pathElement, - delim='.', combine=True)('pathExpression') -litarg = Group( - number | aString -)('args*') -litkwarg = Group(argname + equal + litarg)('kwargs*') +litarg = Group(number | aString)("args*") +litkwarg = Group(argname + equal + litarg)("kwargs*") # lookahead to prevent failing on equals litargs = delimitedList(~litkwarg + litarg) litkwargs = delimitedList(litkwarg) template = Group( - Literal('template') + leftParen + - (call | pathExpression) + - Optional(comma + (litargs | litkwargs)) + - rightParen -)('template') - -if StrictVersion(__version__) >= StrictVersion('2.0.0'): - expression <<= Group(template | call | pathExpression)('expression') + Literal("template") + + leftParen + + (call | pathExpression) + + Optional(comma + (litargs | litkwargs)) + + rightParen +)("template") + +if StrictVersion(__version__) >= StrictVersion("2.0.0"): + expression <<= Group(template | call | pathExpression)("expression") grammar <<= expression else: - expression << (Group(template | call | pathExpression)('expression')) + expression << (Group(template | call | pathExpression)("expression")) grammar << expression diff --git a/graphite_api/storage.py b/graphite_api/storage.py index 1062dcf..1c01ea4 100644 --- a/graphite_api/storage.py +++ b/graphite_api/storage.py @@ -1,5 +1,4 @@ import time - from collections import defaultdict from .intervals import Interval @@ -60,19 +59,23 @@ def __init__(self, pattern, startTime, endTime): self.endTime = endTime self.isExact = is_pattern(pattern) self.interval = Interval( - float('-inf') if startTime is None else startTime, - float('inf') if endTime is None else endTime) + float("-inf") if startTime is None else startTime, + float("inf") if endTime is None else endTime, + ) def __repr__(self): if self.startTime is None: - startString = '*' + startString = "*" else: startString = time.ctime(self.startTime) if self.endTime is None: - endString = '*' + endString = "*" else: endString = time.ctime(self.endTime) - return '' % (self.pattern, startString, - endString) + return "" % ( + self.pattern, + startString, + endString, + ) diff --git a/graphite_api/utils.py b/graphite_api/utils.py index fc2c071..dc17700 100644 --- a/graphite_api/utils.py +++ b/graphite_api/utils.py @@ -15,12 +15,11 @@ import hashlib import pytz - from flask import request def is_pattern(s): - return '*' in s or '?' in s or '[' in s or '{' in s + return "*" in s or "?" in s or "[" in s or "{" in s class RequestParams(object): @@ -76,7 +75,7 @@ def keys(self): def request_json(): - if hasattr(request, 'get_json'): + if hasattr(request, "get_json"): return request.get_json() else: return request.json @@ -90,12 +89,17 @@ def hash_request(): if request.form: keys.update(request.form.keys()) keys.update(request.args.keys()) - params = u",".join([ - u"{0}={1}".format(key, u"&".join(sorted(RequestParams.getlist(key)))) - for key in sorted(keys) if not key.startswith('_') - ]) + params = u",".join( + [ + u"{0}={1}".format( + key, u"&".join(sorted(RequestParams.getlist(key))) + ) + for key in sorted(keys) + if not key.startswith("_") + ] + ) md5 = hashlib.md5() - md5.update(params.encode('utf-8')) + md5.update(params.encode("utf-8")) return md5.hexdigest() diff --git a/setup.cfg b/setup.cfg index 2d139ba..dd99c89 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,3 +5,4 @@ universal = 1 exclude = _vendor,.tox,wsp import-order-style=smarkets application-import-names= +ignore = E203,W503 diff --git a/setup.py b/setup.py index e2cf01f..8d35d82 100644 --- a/setup.py +++ b/setup.py @@ -1,65 +1,66 @@ # coding: utf-8 import sys -from setuptools import setup, find_packages +from setuptools import find_packages, setup install_requires = [ - 'Flask', - 'PyYAML', - 'cairocffi', - 'pyparsing>=1.5.7', - 'pytz', - 'six', - 'structlog', - 'tzlocal', + "Flask", + "PyYAML", + "cairocffi", + "pyparsing>=1.5.7", + "pytz", + "six", + "structlog", + "tzlocal", ] if sys.version_info < (3, 5): - install_requires.append('scandir') + install_requires.append("scandir") -with open('README.rst') as f: +with open("README.rst") as f: long_description = f.read() setup( - name='graphite-api', - version='1.1.3', - url='https://github.com/brutasse/graphite-api', + name="graphite-api", + version="1.1.3", + url="https://github.com/brutasse/graphite-api", author="Bruno Renié, based on Chris Davis's graphite-web", - author_email='bruno@renie.fr', - license='Apache Software License 2.0', - description=('Graphite-web, without the interface. ' - 'Just the rendering HTTP API.'), + author_email="bruno@renie.fr", + license="Apache Software License 2.0", + description=( + "Graphite-web, without the interface. " "Just the rendering HTTP API." + ), long_description=long_description, - packages=find_packages(exclude=['tests']), + packages=find_packages(exclude=["tests"]), include_package_data=True, install_requires=install_requires, extras_require={ - 'sentry': ['raven[flask]'], - 'cyanite': ['cyanite'], - 'cache': ['Flask-Cache'], - 'statsd': ['statsd'], + "sentry": ["raven[flask]"], + "cyanite": ["cyanite"], + "cache": ["Flask-Cache"], + "statsd": ["statsd"], }, zip_safe=False, - platforms='any', + platforms="any", classifiers=( - 'Development Status :: 4 - Beta', - 'Environment :: Web Environment', - 'Framework :: Flask', - 'Intended Audience :: Developers', - 'Intended Audience :: System Administrators', - 'License :: OSI Approved :: Apache Software License', - 'Operating System :: OS Independent', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.2', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', - 'Topic :: Scientific/Engineering :: Information Analysis', - 'Topic :: Scientific/Engineering :: Visualization', - 'Topic :: System :: Monitoring', + "Development Status :: 4 - Beta", + "Environment :: Web Environment", + "Framework :: Flask", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.6", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.2", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", + "Topic :: Scientific/Engineering :: Information Analysis", + "Topic :: Scientific/Engineering :: Visualization", + "Topic :: System :: Monitoring", ), - test_suite='tests', + test_suite="tests", ) diff --git a/tests/__init__.py b/tests/__init__.py index 378e36b..5f726ad 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -4,8 +4,9 @@ from logging.config import dictConfig os.environ.setdefault( - 'GRAPHITE_API_CONFIG', - os.path.join(os.path.dirname(__file__), 'conf.yaml')) # noqa + "GRAPHITE_API_CONFIG", + os.path.join(os.path.dirname(__file__), "conf.yaml"), +) # noqa try: import unittest2 as unittest @@ -18,20 +19,19 @@ from graphite_api.storage import Store -DATA_DIR = '/tmp/graphite-api-data.{0}'.format(os.getpid()) -WHISPER_DIR = os.path.join(DATA_DIR, 'whisper') -SEARCH_INDEX = os.path.join(DATA_DIR, 'index') +DATA_DIR = "/tmp/graphite-api-data.{0}".format(os.getpid()) +WHISPER_DIR = os.path.join(DATA_DIR, "whisper") +SEARCH_INDEX = os.path.join(DATA_DIR, "index") -dictConfig({ - 'version': 1, - 'handlers': { - 'raw': { - 'level': 'DEBUG', - 'class': 'logging.NullHandler', +dictConfig( + { + "version": 1, + "handlers": { + "raw": {"level": "DEBUG", "class": "logging.NullHandler"} }, - }, -}) + } +) class TestCase(unittest.TestCase): @@ -41,9 +41,9 @@ def _cleanup(self): def setUp(self): self._cleanup() os.makedirs(WHISPER_DIR) - app.config['TESTING'] = True - whisper_conf = {'whisper': {'directories': [WHISPER_DIR]}} - app.config['GRAPHITE']['store'] = Store([WhisperFinder(whisper_conf)]) + app.config["TESTING"] = True + whisper_conf = {"whisper": {"directories": [WHISPER_DIR]}} + app.config["GRAPHITE"]["store"] = Store([WhisperFinder(whisper_conf)]) self.app = app.test_client() def tearDown(self): @@ -51,12 +51,13 @@ def tearDown(self): def assertJSON(self, response, data, status_code=200): self.assertEqual(response.status_code, status_code) - self.assertEqual(json.loads(response.data.decode('utf-8')), data) + self.assertEqual(json.loads(response.data.decode("utf-8")), data) def write_series(self, series, retentions=((1, 180),)): file_name = os.path.join( WHISPER_DIR, - '{0}.wsp'.format(series.pathExpression.replace('.', os.sep))) + "{0}.wsp".format(series.pathExpression.replace(".", os.sep)), + ) dir_name = os.path.dirname(file_name) if not os.path.isdir(dir_name): os.makedirs(dir_name) diff --git a/tests/test_attime.py b/tests/test_attime.py index 43fdeb3..ce18967 100644 --- a/tests/test_attime.py +++ b/tests/test_attime.py @@ -1,7 +1,7 @@ import datetime import time -import pytz +import pytz from graphite_api.render.attime import parseATTime from . import TestCase @@ -9,61 +9,59 @@ class AtTestCase(TestCase): default_tz = pytz.utc - specified_tz = pytz.timezone('America/Los_Angeles') + specified_tz = pytz.timezone("America/Los_Angeles") def test_absolute_time(self): - time_string = '12:0020150308' + time_string = "12:0020150308" expected_time = self.default_tz.localize( - datetime.datetime.strptime(time_string, '%H:%M%Y%m%d')) + datetime.datetime.strptime(time_string, "%H:%M%Y%m%d") + ) actual_time = parseATTime(time_string) self.assertEqual(actual_time, expected_time) expected_time = self.specified_tz.localize( - datetime.datetime.strptime(time_string, '%H:%M%Y%m%d')) + datetime.datetime.strptime(time_string, "%H:%M%Y%m%d") + ) actual_time = parseATTime(time_string, self.specified_tz) self.assertEqual(actual_time, expected_time) def test_parse(self): for value in [ str(int(time.time())), - '20140319', - '20130319+1y', - '20130319+1mon', - '20130319+1w', - '12:12_20130319', - '3:05am_20130319', - '3:05pm_20130319', - 'noon20130319', - 'midnight20130319', - 'teatime20130319', - 'yesterday', - 'tomorrow', - '03/19/2014', - '03/19/1800', - '03/19/1950', - 'feb 27', - 'mar 5', - 'mon', - 'tue', - 'wed', - 'thu', - 'fri', - 'sat', - 'sun', - '10:00', - '20160229-1year', - '20160229-4years', - '20160229-1month', - '20130228-1year', - '20130228-4years', - '20130228-1month', + "20140319", + "20130319+1y", + "20130319+1mon", + "20130319+1w", + "12:12_20130319", + "3:05am_20130319", + "3:05pm_20130319", + "noon20130319", + "midnight20130319", + "teatime20130319", + "yesterday", + "tomorrow", + "03/19/2014", + "03/19/1800", + "03/19/1950", + "feb 27", + "mar 5", + "mon", + "tue", + "wed", + "thu", + "fri", + "sat", + "sun", + "10:00", + "20160229-1year", + "20160229-4years", + "20160229-1month", + "20130228-1year", + "20130228-4years", + "20130228-1month", ]: self.assertIsInstance(parseATTime(value), datetime.datetime) - for value in [ - '20130319+1foo', - 'mar', - 'wat', - ]: + for value in ["20130319+1foo", "mar", "wat"]: with self.assertRaises(Exception): parseATTime(value) diff --git a/tests/test_carbonlink.py b/tests/test_carbonlink.py index b4789e4..9351247 100644 --- a/tests/test_carbonlink.py +++ b/tests/test_carbonlink.py @@ -15,17 +15,21 @@ class CarbonLinkTestCase(TestCase): def test_allowed_modules(self): with self.assertRaises(pickle.UnpicklingError) as context: - carbonlink.allowed_module('foo', 'bar') - self.assertIn('Attempting to unpickle unsafe module foo', - str(context.exception)) + carbonlink.allowed_module("foo", "bar") + self.assertIn( + "Attempting to unpickle unsafe module foo", str(context.exception) + ) with self.assertRaises(pickle.UnpicklingError) as context: - carbonlink.allowed_module('__builtin__', 'bar') - self.assertIn('Attempting to unpickle unsafe class bar', - str(context.exception)) + carbonlink.allowed_module("__builtin__", "bar") + self.assertIn( + "Attempting to unpickle unsafe class bar", str(context.exception) + ) - self.assertIsNotNone(carbonlink.allowed_module('collections', 'deque')) - self.assertIsNotNone(carbonlink.allowed_module('__builtin__', 'list')) + self.assertIsNotNone( + carbonlink.allowed_module("collections", "deque") + ) + self.assertIsNotNone(carbonlink.allowed_module("__builtin__", "list")) class ConsistentHashRingTest(TestCase): @@ -36,10 +40,12 @@ def test_chr_compute_ring_position(self): ("127.0.0.1", "cache2"), ] hashring = ConsistentHashRing(hosts) - self.assertEqual(hashring.compute_ring_position('hosts.worker1.cpu'), - 64833) - self.assertEqual(hashring.compute_ring_position('hosts.worker2.cpu'), - 38509) + self.assertEqual( + hashring.compute_ring_position("hosts.worker1.cpu"), 64833 + ) + self.assertEqual( + hashring.compute_ring_position("hosts.worker2.cpu"), 38509 + ) def test_chr_add_node(self): hosts = [ @@ -98,8 +104,8 @@ def test_chr_get_node(self): ("127.0.0.1", "cache2"), ] hashring = ConsistentHashRing(hosts) - node = hashring.get_node('hosts.worker1.cpu') - self.assertEqual(node, ('127.0.0.1', 'cache2')) + node = hashring.get_node("hosts.worker1.cpu") + self.assertEqual(node, ("127.0.0.1", "cache2")) def test_chr_get_nodes(self): hosts = [ @@ -108,7 +114,7 @@ def test_chr_get_nodes(self): ("127.0.0.1", "cache2"), ] hashring = ConsistentHashRing(hosts) - node = hashring.get_nodes('hosts.worker1.cpu') + node = hashring.get_nodes("hosts.worker1.cpu") expected = [ ("127.0.0.1", "cache2"), ("127.0.0.1", "cache0"), @@ -124,11 +130,13 @@ def test_chr_compute_ring_position_fnv1a(self): ("127.0.0.2", "5dd63865534f84899c6e5594dba6749a"), ("127.0.0.3", "866a18b81f2dc4649517a1df13e26f28"), ] - hashring = ConsistentHashRing(hosts, hash_type='fnv1a_ch') - self.assertEqual(hashring.compute_ring_position('hosts.worker1.cpu'), - 59573) - self.assertEqual(hashring.compute_ring_position('hosts.worker2.cpu'), - 35749) + hashring = ConsistentHashRing(hosts, hash_type="fnv1a_ch") + self.assertEqual( + hashring.compute_ring_position("hosts.worker1.cpu"), 59573 + ) + self.assertEqual( + hashring.compute_ring_position("hosts.worker2.cpu"), 35749 + ) def test_chr_get_node_fnv1a(self): hosts = [ @@ -136,31 +144,37 @@ def test_chr_get_node_fnv1a(self): ("127.0.0.2", "5dd63865534f84899c6e5594dba6749a"), ("127.0.0.3", "866a18b81f2dc4649517a1df13e26f28"), ] - hashring = ConsistentHashRing(hosts, hash_type='fnv1a_ch') - self.assertEqual(hashring.get_node('hosts.worker1.cpu'), - ('127.0.0.1', 'ba603c36342304ed77953f84ac4d357b')) - self.assertEqual(hashring.get_node('hosts.worker2.cpu'), - ('127.0.0.3', '866a18b81f2dc4649517a1df13e26f28')) + hashring = ConsistentHashRing(hosts, hash_type="fnv1a_ch") + self.assertEqual( + hashring.get_node("hosts.worker1.cpu"), + ("127.0.0.1", "ba603c36342304ed77953f84ac4d357b"), + ) + self.assertEqual( + hashring.get_node("hosts.worker2.cpu"), + ("127.0.0.3", "866a18b81f2dc4649517a1df13e26f28"), + ) class CarbonLinkPoolTest(TestCase): def test_clp_replication_factor(self): with self.assertRaises(Exception) as context: - CarbonLinkPool(['127.0.0.1:2003'], replication_factor=2) - self.assertIn('replication_factor=2 cannot exceed servers=1', - str(context.exception)) + CarbonLinkPool(["127.0.0.1:2003"], replication_factor=2) + self.assertIn( + "replication_factor=2 cannot exceed servers=1", + str(context.exception), + ) def test_clp_requests(self): hosts = [ - '10.0.0.1:2003:cache0', - '10.0.0.2:2003:cache1', - '10.0.0.3:2003:cache2', + "10.0.0.1:2003:cache0", + "10.0.0.2:2003:cache1", + "10.0.0.3:2003:cache2", ] carbonlink = CarbonLinkPool(hosts, replication_factor=3) - with patch('socket.socket'): + with patch("socket.socket"): for host in hosts: - server, port, instance = host.split(':') + server, port, instance = host.split(":") conn = carbonlink.get_connection((server, instance)) conn.connect.assert_called_with((server, int(port))) carbonlink.connections[(server, instance)].add(conn) @@ -168,41 +182,41 @@ def test_clp_requests(self): def mock_recv_query(size): data = pickle.dumps(dict(datapoints=[1, 2, 3])) if size == 4: - return struct.pack('!I', len(data)) + return struct.pack("!I", len(data)) elif size == len(data): return data else: - raise ValueError('unexpected size %s' % size) + raise ValueError("unexpected size %s" % size) conn.recv.side_effect = mock_recv_query - datapoints = carbonlink.query('hosts.worker1.cpu') + datapoints = carbonlink.query("hosts.worker1.cpu") self.assertEqual(datapoints, [1, 2, 3]) - datapoints = carbonlink.query('carbon.send_to_all.request') + datapoints = carbonlink.query("carbon.send_to_all.request") self.assertEqual(datapoints, [1, 2, 3] * 3) def mock_recv_get_metadata(size): - data = pickle.dumps(dict(value='foo')) + data = pickle.dumps(dict(value="foo")) if size == 4: - return struct.pack('!I', len(data)) + return struct.pack("!I", len(data)) elif size == len(data): return data else: - raise ValueError('unexpected size %s' % size) + raise ValueError("unexpected size %s" % size) conn.recv.side_effect = mock_recv_get_metadata - metadata = carbonlink.get_metadata('hosts.worker1.cpu', 'key') - self.assertEqual(metadata, 'foo') + metadata = carbonlink.get_metadata("hosts.worker1.cpu", "key") + self.assertEqual(metadata, "foo") def mock_recv_set_metadata(size): - data = pickle.dumps(dict(old_value='foo', new_value='bar')) + data = pickle.dumps(dict(old_value="foo", new_value="bar")) if size == 4: - return struct.pack('!I', len(data)) + return struct.pack("!I", len(data)) elif size == len(data): return data else: - raise ValueError('unexpected size %s' % size) + raise ValueError("unexpected size %s" % size) conn.recv.side_effect = mock_recv_set_metadata - results = carbonlink.set_metadata('hosts.worker1.cpu', 'foo', 'bar') - self.assertEqual(results, {'old_value': 'foo', 'new_value': 'bar'}) + results = carbonlink.set_metadata("hosts.worker1.cpu", "foo", "bar") + self.assertEqual(results, {"old_value": "foo", "new_value": "bar"}) diff --git a/tests/test_finders.py b/tests/test_finders.py index 242cbbd..47c9c07 100644 --- a/tests/test_finders.py +++ b/tests/test_finders.py @@ -20,17 +20,16 @@ class FinderTest(TestCase): - def test_custom_finder(self): store = Store([DummyFinder()]) nodes = list(store.find("foo")) self.assertEqual(len(nodes), 1) - self.assertEqual(nodes[0].path, 'foo') + self.assertEqual(nodes[0].path, "foo") - nodes = list(store.find('bar.*')) + nodes = list(store.find("bar.*")) self.assertEqual(len(nodes), 10) node = nodes[0] - self.assertEqual(node.path.split('.')[0], 'bar') + self.assertEqual(node.path.split(".")[0], "bar") time_info, series = node.fetch(100, 200) self.assertEqual(time_info, (100, 200, 10)) @@ -40,12 +39,12 @@ def test_multi_finder(self): store = Store([DummyFinder(), DummyFinder()]) nodes = list(store.find("foo")) self.assertEqual(len(nodes), 1) - self.assertEqual(nodes[0].path, 'foo') + self.assertEqual(nodes[0].path, "foo") - nodes = list(store.find('bar.*')) + nodes = list(store.find("bar.*")) self.assertEqual(len(nodes), 10) node = nodes[0] - self.assertEqual(node.path.split('.')[0], 'bar') + self.assertEqual(node.path.split(".")[0], "bar") time_info, series = node.fetch(100, 200) self.assertEqual(time_info, (100, 200, 10)) @@ -53,16 +52,17 @@ def test_multi_finder(self): class DummyReader(object): - __slots__ = ('path',) + __slots__ = ("path",) def __init__(self, path): self.path = path def fetch(self, start_time, end_time): npoints = (end_time - start_time) // 10 - return (start_time, end_time, 10), [ - random.choice([None, 1, 2, 3]) for i in range(npoints) - ] + return ( + (start_time, end_time, 10), + [random.choice([None, 1, 2, 3]) for i in range(npoints)], + ) def get_intervals(self): return IntervalSet([Interval(time.time() - 3600, time.time())]) @@ -70,26 +70,25 @@ def get_intervals(self): class DummyFinder(object): def find_nodes(self, query): - if query.pattern == 'foo': - yield BranchNode('foo') + if query.pattern == "foo": + yield BranchNode("foo") - elif query.pattern == 'bar.*': + elif query.pattern == "bar.*": for i in range(10): - path = 'bar.{0}'.format(i) + path = "bar.{0}".format(i) yield LeafNode(path, DummyReader(path)) class WhisperFinderTest(TestCase): - def scandir_mock(d): return scandir(d) - @patch('graphite_api.finders.whisper.scandir', wraps=scandir_mock) + @patch("graphite_api.finders.whisper.scandir", wraps=scandir_mock) def test_whisper_finder(self, scandir_mocked): for db in ( - ('whisper_finder', 'foo.wsp'), - ('whisper_finder', 'foo', 'bar', 'baz.wsp'), - ('whisper_finder', 'bar', 'baz', 'baz.wsp'), + ("whisper_finder", "foo.wsp"), + ("whisper_finder", "foo", "bar", "baz.wsp"), + ("whisper_finder", "bar", "baz", "baz.wsp"), ): db_path = os.path.join(WHISPER_DIR, *db) if not os.path.exists(os.path.dirname(db_path)): @@ -97,95 +96,95 @@ def test_whisper_finder(self, scandir_mocked): whisper.create(db_path, [(1, 60)]) try: - store = app.config['GRAPHITE']['store'] + store = app.config["GRAPHITE"]["store"] scandir_mocked.call_count = 0 - nodes = store.find('whisper_finder.foo') + nodes = store.find("whisper_finder.foo") self.assertEqual(len(list(nodes)), 2) self.assertEqual(scandir_mocked.call_count, 0) scandir_mocked.call_count = 0 - nodes = store.find('whisper_finder.foo.bar.baz') + nodes = store.find("whisper_finder.foo.bar.baz") self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mocked.call_count, 0) scandir_mocked.call_count = 0 - nodes = store.find('whisper_finder.*.ba?.{baz,foo}') + nodes = store.find("whisper_finder.*.ba?.{baz,foo}") self.assertEqual(len(list(nodes)), 2) self.assertEqual(scandir_mocked.call_count, 5) scandir_mocked.call_count = 0 - nodes = store.find('whisper_finder.{foo,bar}.{baz,bar}.{baz,foo}') + nodes = store.find("whisper_finder.{foo,bar}.{baz,bar}.{baz,foo}") self.assertEqual(len(list(nodes)), 2) self.assertEqual(scandir_mocked.call_count, 5) scandir_mocked.call_count = 0 - nodes = store.find('whisper_finder.{foo}.bar.*') + nodes = store.find("whisper_finder.{foo}.bar.*") self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mocked.call_count, 2) scandir_mocked.call_count = 0 - nodes = store.find('whisper_finder.foo.{ba{r,z},baz}.baz') + nodes = store.find("whisper_finder.foo.{ba{r,z},baz}.baz") self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mocked.call_count, 1) scandir_mocked.call_count = 0 - nodes = store.find('whisper_finder.{foo,garbage}.bar.baz') + nodes = store.find("whisper_finder.{foo,garbage}.bar.baz") self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mocked.call_count, 1) scandir_mocked.call_count = 0 - nodes = store.find('whisper_finder.{fo{o}}.bar.baz') + nodes = store.find("whisper_finder.{fo{o}}.bar.baz") self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mocked.call_count, 1) scandir_mocked.call_count = 0 - nodes = store.find('whisper_finder.foo{}.bar.baz') + nodes = store.find("whisper_finder.foo{}.bar.baz") self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mocked.call_count, 1) scandir_mocked.call_count = 0 - nodes = store.find('whisper_finder.{fo,ba}{o}.bar.baz') + nodes = store.find("whisper_finder.{fo,ba}{o}.bar.baz") self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mocked.call_count, 1) scandir_mocked.call_count = 0 - nodes = store.find('whisper_finder.{fo,ba}{o,o}.bar.baz') + nodes = store.find("whisper_finder.{fo,ba}{o,o}.bar.baz") self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mocked.call_count, 1) scandir_mocked.call_count = 0 - nodes = store.find('whisper_finder.{fo,ba}{o,z}.bar.baz') + nodes = store.find("whisper_finder.{fo,ba}{o,z}.bar.baz") self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mocked.call_count, 1) finally: scandir_mocked.call_count = 0 - @patch('graphite_api.finders.whisper.scandir', wraps=scandir_mock) + @patch("graphite_api.finders.whisper.scandir", wraps=scandir_mock) def test_gzipped_whisper_finder(self, scandir_mocked): for db in ( - ('gzwhisper_finder', 'foo.wsp'), - ('gzwhisper_finder', 'foo', 'bar', 'baz.wsp'), - ('gzwhisper_finder', 'bar', 'baz', 'baz.wsp'), + ("gzwhisper_finder", "foo.wsp"), + ("gzwhisper_finder", "foo", "bar", "baz.wsp"), + ("gzwhisper_finder", "bar", "baz", "baz.wsp"), ): db_path = os.path.join(WHISPER_DIR, *db) if not os.path.exists(os.path.dirname(db_path)): os.makedirs(os.path.dirname(db_path)) whisper.create(db_path, [(1, 60)]) - with open(db_path, 'rb') as f_in: - f_out = gzip.open("%s.gz" % db_path, 'wb') + with open(db_path, "rb") as f_in: + f_out = gzip.open("%s.gz" % db_path, "wb") shutil.copyfileobj(f_in, f_out) f_out.close() os.remove(db_path) try: - store = app.config['GRAPHITE']['store'] + store = app.config["GRAPHITE"]["store"] scandir_mocked.call_count = 0 - nodes = store.find('gzwhisper_finder.foo') + nodes = store.find("gzwhisper_finder.foo") self.assertEqual(len(list(nodes)), 2) self.assertEqual(scandir_mocked.call_count, 0) scandir_mocked.call_count = 0 - nodes = store.find('gzwhisper_finder.foo{}.bar.baz') + nodes = store.find("gzwhisper_finder.foo{}.bar.baz") self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mocked.call_count, 1) @@ -193,7 +192,7 @@ def test_gzipped_whisper_finder(self, scandir_mocked): scandir_mocked.call_count = 0 def test_globstar(self): - store = app.config['GRAPHITE']['store'] + store = app.config["GRAPHITE"]["store"] query = "x.**.x" hits = ["x.x", "x._.x", "x._._.x"] misses = ["x.x.o", "o.x.x", "x._.x._.o", "o._.x._.x"] @@ -201,7 +200,7 @@ def test_globstar(self): db_path = os.path.join(WHISPER_DIR, path.replace(".", os.sep)) if not os.path.exists(os.path.dirname(db_path)): os.makedirs(os.path.dirname(db_path)) - whisper.create(db_path + '.wsp', [(1, 60)]) + whisper.create(db_path + ".wsp", [(1, 60)]) paths = [node.path for node in store.find(query, local=True)] for hit in hits: @@ -210,21 +209,31 @@ def test_globstar(self): self.assertNotIn(miss, paths) def test_multiple_globstars(self): - store = app.config['GRAPHITE']['store'] + store = app.config["GRAPHITE"]["store"] query = "y.**.y.**.y" hits = [ - "y.y.y", "y._.y.y", "y.y._.y", "y._.y._.y", - "y._._.y.y", "y.y._._.y" + "y.y.y", + "y._.y.y", + "y.y._.y", + "y._.y._.y", + "y._._.y.y", + "y.y._._.y", ] misses = [ - "y.o.y", "o.y.y", "y.y.o", "o.y.y.y", "y.y.y.o", - "o._.y._.y", "y._.o._.y", "y._.y._.o" + "y.o.y", + "o.y.y", + "y.y.o", + "o.y.y.y", + "y.y.y.o", + "o._.y._.y", + "y._.o._.y", + "y._.y._.o", ] for path in hits + misses: db_path = os.path.join(WHISPER_DIR, path.replace(".", os.sep)) if not os.path.exists(os.path.dirname(db_path)): os.makedirs(os.path.dirname(db_path)) - whisper.create(db_path + '.wsp', [(1, 60)]) + whisper.create(db_path + ".wsp", [(1, 60)]) paths = [node.path for node in store.find(query, local=True)] for hit in hits: @@ -233,7 +242,7 @@ def test_multiple_globstars(self): self.assertNotIn(miss, paths) def test_terminal_globstar(self): - store = app.config['GRAPHITE']['store'] + store = app.config["GRAPHITE"]["store"] query = "z.**" hits = ["z._", "z._._", "z._._._"] misses = ["z", "o._", "o.z._", "o._.z"] @@ -241,7 +250,7 @@ def test_terminal_globstar(self): db_path = os.path.join(WHISPER_DIR, path.replace(".", os.sep)) if not os.path.exists(os.path.dirname(db_path)): os.makedirs(os.path.dirname(db_path)) - whisper.create(db_path + '.wsp', [(1, 60)]) + whisper.create(db_path + ".wsp", [(1, 60)]) paths = [node.path for node in store.find(query, local=True)] for hit in hits: diff --git a/tests/test_functions.py b/tests/test_functions.py index f1ae964..481fdaf 100644 --- a/tests/test_functions.py +++ b/tests/test_functions.py @@ -1,16 +1,13 @@ import copy import math import time - from datetime import datetime -import pytz - try: from unittest.mock import patch, call, MagicMock except ImportError: from mock import patch, call, MagicMock - +import pytz from graphite_api import functions from graphite_api.app import app from graphite_api.render.attime import parseATTime @@ -170,7 +167,9 @@ def test_safe_stddev_empty_list(self): self.assertEqual(functions.safeStdDev([]), None) def test_safe_stddev_all_numbers(self): - self.assertEqual(functions.safeStdDev([1, 2, 3, 4]), 1.118033988749895) + self.assertEqual( + functions.safeStdDev([1, 2, 3, 4]), 1.118033988749895 + ) def test_safe_stddev_all_None(self): self.assertEqual(functions.safeStdDev([None, None, None, None]), None) @@ -258,8 +257,9 @@ def test_safe_map_all_numbers(self): self.assertEqual(functions.safeMap(abs, [1, 2, 3, 4]), [1, 2, 3, 4]) def test_safe_map_all_None(self): - self.assertEqual(functions.safeMap(abs, [None, None, None, None]), - None) + self.assertEqual( + functions.safeMap(abs, [None, None, None, None]), None + ) def test_safe_map_mixed(self): self.assertEqual(functions.safeMap(abs, [10, None, 5, None]), [10, 5]) @@ -305,25 +305,37 @@ def test_normalize_empty(self): def test_normalize_None_values(self): seriesList = [] - seriesList.append(TimeSeries("collectd.test-db{0}.load.value", 0, 5, 1, - [None, None, None, None, None])) - self.assertEqual(functions.normalize([seriesList]), - (seriesList, 0, 5, 1)) + seriesList.append( + TimeSeries( + "collectd.test-db{0}.load.value", + 0, + 5, + 1, + [None, None, None, None, None], + ) + ) + self.assertEqual( + functions.normalize([seriesList]), (seriesList, 0, 5, 1) + ) def test_normalize_generate_series_list_input(self): - seriesList = self._generate_series_list(config=[range(101), - range(101)]) - self.assertEqual(functions.normalize([seriesList]), - (seriesList, seriesList[0].start, - seriesList[0].end, 1)) + seriesList = self._generate_series_list( + config=[range(101), range(101)] + ) + self.assertEqual( + functions.normalize([seriesList]), + (seriesList, seriesList[0].start, seriesList[0].end, 1), + ) # Test matchSeries() def test_match_series_assert(self): seriesList = self._generate_series_list() with self.assertRaises(AssertionError) as context: functions.matchSeries(seriesList[0], []) - self.assertIn('The number of series in each argument ' - 'must be the same', str(context.exception)) + self.assertIn( + "The number of series in each argument " "must be the same", + str(context.exception), + ) def test_match_series_empty(self): results = functions.matchSeries([], []) @@ -333,45 +345,67 @@ def test_match_series_empty(self): def test_match_series(self): seriesList1 = [ - TimeSeries('collectd.test-db3.load.value', 0, 1, 1, [3, 30, 31]), - TimeSeries('collectd.test-db1.load.value', 0, 1, 1, [1, 10, 11]), - TimeSeries('collectd.test-db2.load.value', 0, 1, 1, [2, 20, 21]), - TimeSeries('collectd.test-db4.load.value', 0, 1, 1, [4, 40, 41]), + TimeSeries("collectd.test-db3.load.value", 0, 1, 1, [3, 30, 31]), + TimeSeries("collectd.test-db1.load.value", 0, 1, 1, [1, 10, 11]), + TimeSeries("collectd.test-db2.load.value", 0, 1, 1, [2, 20, 21]), + TimeSeries("collectd.test-db4.load.value", 0, 1, 1, [4, 40, 41]), ] seriesList2 = [ - TimeSeries('collectd.test-db4.load.value', 0, 1, 1, [4, 8, 12]), - TimeSeries('collectd.test-db3.load.value', 0, 1, 1, [3, 7, 11]), - TimeSeries('collectd.test-db1.load.value', 0, 1, 1, [1, 5, 9]), - TimeSeries('collectd.test-db2.load.value', 0, 1, 1, [2, 6, 10]), + TimeSeries("collectd.test-db4.load.value", 0, 1, 1, [4, 8, 12]), + TimeSeries("collectd.test-db3.load.value", 0, 1, 1, [3, 7, 11]), + TimeSeries("collectd.test-db1.load.value", 0, 1, 1, [1, 5, 9]), + TimeSeries("collectd.test-db2.load.value", 0, 1, 1, [2, 6, 10]), ] - expectedResult = [[ - TimeSeries('collectd.test-db1.load.value', 0, 1, 1, [1, 10, 11]), - TimeSeries('collectd.test-db2.load.value', 0, 1, 1, [2, 20, 21]), - TimeSeries('collectd.test-db3.load.value', 0, 1, 1, [3, 30, 31]), - TimeSeries('collectd.test-db4.load.value', 0, 1, 1, [4, 40, 41]), - ], [ - TimeSeries('collectd.test-db1.load.value', 0, 1, 1, [1, 5, 9]), - TimeSeries('collectd.test-db2.load.value', 0, 1, 1, [2, 6, 10]), - TimeSeries('collectd.test-db3.load.value', 0, 1, 1, [3, 7, 11]), - TimeSeries('collectd.test-db4.load.value', 0, 1, 1, [4, 8, 12]), - ]] - results = functions.matchSeries(copy.deepcopy(seriesList1), - copy.deepcopy(seriesList2)) + expectedResult = [ + [ + TimeSeries( + "collectd.test-db1.load.value", 0, 1, 1, [1, 10, 11] + ), + TimeSeries( + "collectd.test-db2.load.value", 0, 1, 1, [2, 20, 21] + ), + TimeSeries( + "collectd.test-db3.load.value", 0, 1, 1, [3, 30, 31] + ), + TimeSeries( + "collectd.test-db4.load.value", 0, 1, 1, [4, 40, 41] + ), + ], + [ + TimeSeries( + "collectd.test-db1.load.value", 0, 1, 1, [1, 5, 9] + ), + TimeSeries( + "collectd.test-db2.load.value", 0, 1, 1, [2, 6, 10] + ), + TimeSeries( + "collectd.test-db3.load.value", 0, 1, 1, [3, 7, 11] + ), + TimeSeries( + "collectd.test-db4.load.value", 0, 1, 1, [4, 8, 12] + ), + ], + ] + results = functions.matchSeries( + copy.deepcopy(seriesList1), copy.deepcopy(seriesList2) + ) for i, (series1, series2) in enumerate(results): self.assertEqual(series1, expectedResult[0][i]) self.assertEqual(series2, expectedResult[1][i]) # Test formatPathExpressions() def test_format_path_expressions_empty_list(self): - self.assertEqual(functions.formatPathExpressions([]), '') + self.assertEqual(functions.formatPathExpressions([]), "") def test_format_path_expressions(self): seriesList = self._generate_series_list() - self.assertEqual(functions.formatPathExpressions(seriesList), - "collectd.test-db1.load.value," - "collectd.test-db2.load.value," - "collectd.test-db3.load.value," - "collectd.test-db4.load.value") + self.assertEqual( + functions.formatPathExpressions(seriesList), + "collectd.test-db1.load.value," + "collectd.test-db2.load.value," + "collectd.test-db3.load.value," + "collectd.test-db4.load.value", + ) def test_highest_max(self): config = [20, 50, 30, 40] @@ -412,9 +446,13 @@ def testGetPercentile(self): series, expected = conf result = functions._getPercentile(series, 30) self.assertEqual( - expected, result, - ('For series index <%s> the 30th percentile ordinal is not ' - '%d, but %d ' % (index, expected, result))) + expected, + result, + ( + "For series index <%s> the 30th percentile ordinal is not " + "%d, but %d " % (index, expected, result) + ), + ) def test_n_percentile(self): config = [ @@ -433,23 +471,36 @@ def n_percentile(perc, expect): seriesList = [] expected = [] for i, c in enumerate(config): - seriesList.append(TimeSeries('Test(%d)' % i, 0, len(c), 1, c)) - expected.append(TimeSeries('nPercentile(Test(%d), %d)' % - (i, perc), 0, len(c), 1, - expect[i] * len(c))) + seriesList.append(TimeSeries("Test(%d)" % i, 0, len(c), 1, c)) + expected.append( + TimeSeries( + "nPercentile(Test(%d), %d)" % (i, perc), + 0, + len(c), + 1, + expect[i] * len(c), + ) + ) result = functions.nPercentile({}, seriesList, perc) self.assertEqual(expected, result) n_percentile(30, [[20], [31], [61], [91], [30], [60], [90], [90]]) - n_percentile(90, [[50], [91], [181], [271], [90], [180], [270], [270]]) - n_percentile(95, [[50], [96], [191], [286], [95], [190], [285], [285]]) - - def _generate_series_list(self, config=( - range(101), - range(2, 103), - [1] * 2 + [None] * 90 + [1] * 2 + [None] * 7, - [] - )): + n_percentile( + 90, [[50], [91], [181], [271], [90], [180], [270], [270]] + ) + n_percentile( + 95, [[50], [96], [191], [286], [95], [190], [285], [285]] + ) + + def _generate_series_list( + self, + config=( + range(101), + range(2, 103), + [1] * 2 + [None] * 90 + [1] * 2 + [None] * 7, + [], + ), + ): seriesList = [] now = int(time.time()) @@ -497,8 +548,11 @@ def test_limit(self): seriesList = self._generate_series_list() limit = len(seriesList) - 1 results = functions.limit({}, seriesList, limit) - self.assertEqual(len(results), limit, - "More than {0} results returned".format(limit)) + self.assertEqual( + len(results), + limit, + "More than {0} results returned".format(limit), + ) def _verify_series_options(self, seriesList, name, value): """ @@ -525,41 +579,58 @@ def test_draw_as_infinite(self): self._verify_series_options(results, "drawAsInfinite", True) def test_vertical_line(self): - result = functions.verticalLine({ - 'startTime': datetime(1970, 1, 1, 1, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 1, 1, 1, 2, 0, 0, pytz.utc), - 'tzinfo': pytz.timezone('UTC'), - }, "01:0019700101", "foo") - expectedResult = [TimeSeries('foo', 3600, 3600, 1.0, [1.0, 1.0])] - expectedResult[0].options = {'drawAsInfinite': True} + result = functions.verticalLine( + { + "startTime": datetime(1970, 1, 1, 1, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 1, 1, 1, 2, 0, 0, pytz.utc), + "tzinfo": pytz.timezone("UTC"), + }, + "01:0019700101", + "foo", + ) + expectedResult = [TimeSeries("foo", 3600, 3600, 1.0, [1.0, 1.0])] + expectedResult[0].options = {"drawAsInfinite": True} self.assertEqual(result, expectedResult) def test_vertical_line_color(self): - result = functions.verticalLine({ - 'startTime': datetime(1970, 1, 1, 1, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 1, 1, 1, 2, 0, 0, pytz.utc), - 'tzinfo': pytz.timezone('UTC'), - }, "01:0019700101", "foo", "white") - expectedResult = [TimeSeries('foo', 3600, 3600, 1.0, [1.0, 1.0])] - expectedResult[0].options = {'drawAsInfinite': True} + result = functions.verticalLine( + { + "startTime": datetime(1970, 1, 1, 1, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 1, 1, 1, 2, 0, 0, pytz.utc), + "tzinfo": pytz.timezone("UTC"), + }, + "01:0019700101", + "foo", + "white", + ) + expectedResult = [TimeSeries("foo", 3600, 3600, 1.0, [1.0, 1.0])] + expectedResult[0].options = {"drawAsInfinite": True} expectedResult[0].color = "white" self.assertEqual(result, expectedResult) def test_vertical_line_before_start(self): with self.assertRaises(ValueError): - functions.verticalLine({ - 'startTime': datetime(1971, 1, 1, 1, 0, 0, 0, pytz.utc), - 'endTime': datetime(1971, 1, 1, 1, 2, 0, 0, pytz.utc), - 'tzinfo': pytz.timezone('UTC'), - }, "01:0019700101", "foo") + functions.verticalLine( + { + "startTime": datetime(1971, 1, 1, 1, 0, 0, 0, pytz.utc), + "endTime": datetime(1971, 1, 1, 1, 2, 0, 0, pytz.utc), + "tzinfo": pytz.timezone("UTC"), + }, + "01:0019700101", + "foo", + ) def test_vertical_line_after_end(self): with self.assertRaises(ValueError): - functions.verticalLine({ - 'startTime': datetime(1970, 1, 1, 1, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 1, 1, 1, 2, 0, 0, pytz.utc), - 'tzinfo': pytz.timezone('UTC'), - }, "01:0019710101", "foo") + functions.verticalLine( + { + "startTime": datetime(1970, 1, 1, 1, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 1, 1, 1, 2, 0, 0, pytz.utc), + "tzinfo": pytz.timezone("UTC"), + }, + "01:0019710101", + "foo", + ) def test_line_width(self): seriesList = self._generate_series_list() @@ -570,24 +641,31 @@ def test_line_width(self): def test_transform_null(self): seriesList = self._generate_series_list() transform = -5 - results = functions.transformNull({}, copy.deepcopy(seriesList), - transform) + results = functions.transformNull( + {}, copy.deepcopy(seriesList), transform + ) for counter, series in enumerate(seriesList): if None not in series: continue # If the None values weren't transformed, there is a problem - self.assertNotIn(None, results[counter], - "tranformNull should remove all None values") + self.assertNotIn( + None, + results[counter], + "tranformNull should remove all None values", + ) # Anywhere a None was in the original series, verify it # was transformed to the given value it should be. for i, value in enumerate(series): if value is None: result_val = results[counter][i] self.assertEqual( - transform, result_val, + transform, + result_val, "Transformed value should be {0}, not {1}".format( - transform, result_val)) + transform, result_val + ), + ) def test_transform_null_reference(self): seriesList = self._generate_series_list() @@ -597,8 +675,9 @@ def test_transform_null_reference(self): if index % 2 != 0: referenceSeries[index] = None - results = functions.transformNull({}, copy.deepcopy(seriesList), - transform, [referenceSeries]) + results = functions.transformNull( + {}, copy.deepcopy(seriesList), transform, [referenceSeries] + ) for counter, series in enumerate(seriesList): if None not in series: @@ -610,55 +689,127 @@ def test_transform_null_reference(self): if value is None and referenceSeries[i] is not None: result_val = results[counter][i] self.assertEqual( - transform, result_val, + transform, + result_val, "Transformed value should be {0}, not {1}".format( - transform, result_val)) + transform, result_val + ), + ) def test_transform_null_reference_empty(self): seriesList = self._generate_series_list() transform = -5 referenceSeries = [] - results = functions.transformNull({}, copy.deepcopy(seriesList), - transform, [referenceSeries]) + results = functions.transformNull( + {}, copy.deepcopy(seriesList), transform, [referenceSeries] + ) for counter, series in enumerate(seriesList): if None not in series: continue # If the None values weren't transformed, there is a problem - self.assertNotIn(None, results[counter], - "tranformNull should remove all None values") + self.assertNotIn( + None, + results[counter], + "tranformNull should remove all None values", + ) # Anywhere a None was in the original series, verify it # was transformed to the given value if a value existed for i, value in enumerate(series): if value is None: result_val = results[counter][i] self.assertEqual( - transform, result_val, + transform, + result_val, "Transformed value should be {0}, not {1}".format( - transform, result_val)) + transform, result_val + ), + ) def test_group(self): seriesList = [ - TimeSeries('collectd.test-db1.load.value', - 0, 600, 60, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), - TimeSeries('collectd.test-db2.load.value', - 0, 600, 60, [None] * 10), - TimeSeries('collectd.test-db3.load.value', - 0, 600, 60, [1, 2, None, None, None, 6, 7, 8, 9, 10]), - TimeSeries('collectd.test-db4.load.value', - 0, 600, 60, [1, 2, 3, 4, 5, 6, 7, 8, 9, None]), + TimeSeries( + "collectd.test-db1.load.value", + 0, + 600, + 60, + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + ), + TimeSeries( + "collectd.test-db2.load.value", 0, 600, 60, [None] * 10 + ), + TimeSeries( + "collectd.test-db3.load.value", + 0, + 600, + 60, + [1, 2, None, None, None, 6, 7, 8, 9, 10], + ), + TimeSeries( + "collectd.test-db4.load.value", + 0, + 600, + 60, + [1, 2, 3, 4, 5, 6, 7, 8, 9, None], + ), ] for series in seriesList: series.pathExpression = series.name request_context = {} - result = functions.group(request_context, seriesList[0], - seriesList[1], seriesList[2], seriesList[3]) - self.assertEqual(result, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, None, None, - None, None, None, None, None, None, None, - None, 1, 2, None, None, None, 6, 7, 8, 9, - 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, None]) + result = functions.group( + request_context, + seriesList[0], + seriesList[1], + seriesList[2], + seriesList[3], + ) + self.assertEqual( + result, + [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + 1, + 2, + None, + None, + None, + 6, + 7, + 8, + 9, + 10, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + None, + ], + ) def test_alias(self): seriesList = self._generate_series_list() @@ -675,7 +826,9 @@ def test_alias_sub(self): self.assertTrue( series.name.startswith(substitution), "aliasSub should replace the name with {0}".format( - substitution)) + substitution + ), + ) # TODO: Add tests for * globbing and {} matching to this def test_alias_by_node(self): @@ -689,10 +842,10 @@ def verify_node_name(cases, expected, *nodes): if expected: expected_name = expected[i] else: - fragments = seriesList[i].name.split('.') + fragments = seriesList[i].name.split(".") # Super simplistic. Doesn't match {thing1,thing2} # or glob with *, both of what graphite allow you to use - expected_name = '.'.join([fragments[i] for i in nodes]) + expected_name = ".".join([fragments[i] for i in nodes]) self.assertEqual(series.name, expected_name) verify_node_name(seriesList, None, 1) @@ -761,19 +914,19 @@ def test_average_series_wildcards_empty_series_int_position(self): def test_average_series_wildcards(self): series = self._generate_series_list() average = functions.averageSeriesWithWildcards({}, series, 1)[0] - self.assertEqual(average[:3], [1.0, 5/3., 3.0]) - self.assertEqual(average.name, 'collectd.load.value') + self.assertEqual(average[:3], [1.0, 5 / 3.0, 3.0]) + self.assertEqual(average.name, "collectd.load.value") def _generate_mr_series(self): seriesList = [ - TimeSeries('group.server1.metric1', 0, 1, 1, [None]), - TimeSeries('group.server1.metric2', 0, 1, 1, [None]), - TimeSeries('group.server2.metric1', 0, 1, 1, [None]), - TimeSeries('group.server2.metric2', 0, 1, 1, [None]), + TimeSeries("group.server1.metric1", 0, 1, 1, [None]), + TimeSeries("group.server1.metric2", 0, 1, 1, [None]), + TimeSeries("group.server2.metric1", 0, 1, 1, [None]), + TimeSeries("group.server2.metric2", 0, 1, 1, [None]), ] mappedResult = [ [seriesList[0], seriesList[1]], - [seriesList[2], seriesList[3]] + [seriesList[2], seriesList[3]], ] return seriesList, mappedResult @@ -785,40 +938,53 @@ def test_mapSeries(self): def test_reduceSeries(self): sl, inputList = self._generate_mr_series() expectedResult = [ - TimeSeries('group.server2.reduce.mock', 0, 1, 1, [None]), - TimeSeries('group.server2.reduce.mock', 0, 1, 1, [None]) + TimeSeries("group.server2.reduce.mock", 0, 1, 1, [None]), + TimeSeries("group.server2.reduce.mock", 0, 1, 1, [None]), ] - resultSeriesList = [TimeSeries('mock(series)', 0, 1, 1, [None])] + resultSeriesList = [TimeSeries("mock(series)", 0, 1, 1, [None])] mock = MagicMock(return_value=resultSeriesList) - with patch.dict(app.config['GRAPHITE']['functions'], {'mock': mock}): - results = functions.reduceSeries({}, copy.deepcopy(inputList), - "mock", 2, "metric1", "metric2") + with patch.dict(app.config["GRAPHITE"]["functions"], {"mock": mock}): + results = functions.reduceSeries( + {}, copy.deepcopy(inputList), "mock", 2, "metric1", "metric2" + ) self.assertEqual(results, expectedResult) - self.assertEqual(mock.mock_calls, [ - call({}, *[[x] for x in inputList[0]]), - call({}, *[[x] for x in inputList[1]]), - ]) + self.assertEqual( + mock.mock_calls, + [ + call({}, *[[x] for x in inputList[0]]), + call({}, *[[x] for x in inputList[1]]), + ], + ) def test_reduceSeries_asPercent(self): seriesList = [ - TimeSeries('group.server1.bytes_used', 0, 1, 1, [1]), - TimeSeries('group.server1.total_bytes', 0, 1, 1, [2]), - TimeSeries('group.server2.bytes_used', 0, 1, 1, [3]), - TimeSeries('group.server2.total_bytes', 0, 1, 1, [4]), + TimeSeries("group.server1.bytes_used", 0, 1, 1, [1]), + TimeSeries("group.server1.total_bytes", 0, 1, 1, [2]), + TimeSeries("group.server2.bytes_used", 0, 1, 1, [3]), + TimeSeries("group.server2.total_bytes", 0, 1, 1, [4]), ] for series in seriesList: series.pathExpression = "tempPath" expectedResult = [ # 50 == 100 * 1 / 2 - TimeSeries('group.server1.reduce.asPercent', 0, 1, 1, [50]), + TimeSeries("group.server1.reduce.asPercent", 0, 1, 1, [50]), # 100 * 3 / 4 - TimeSeries('group.server2.reduce.asPercent', 0, 1, 1, [75]), + TimeSeries("group.server2.reduce.asPercent", 0, 1, 1, [75]), ] mappedResult = ( - [seriesList[0]], [seriesList[1]], [seriesList[2]], [seriesList[3]]) + [seriesList[0]], + [seriesList[1]], + [seriesList[2]], + [seriesList[3]], + ) results = functions.reduceSeries( - {}, copy.deepcopy(mappedResult), - "asPercent", 2, "bytes_used", "total_bytes") + {}, + copy.deepcopy(mappedResult), + "asPercent", + 2, + "bytes_used", + "total_bytes", + ) self.assertEqual(results, expectedResult) def test_pow(self): @@ -840,11 +1006,13 @@ def test_sum_series_empty(self): def test_sum_series(self): series = self._generate_series_list() [sum_] = functions.sumSeries({}, series) - self.assertEqual(sum_.pathExpression, - "sumSeries(collectd.test-db1.load.value," - "collectd.test-db2.load.value," - "collectd.test-db3.load.value," - "collectd.test-db4.load.value)") + self.assertEqual( + sum_.pathExpression, + "sumSeries(collectd.test-db1.load.value," + "collectd.test-db2.load.value," + "collectd.test-db3.load.value," + "collectd.test-db4.load.value)", + ) self.assertEqual(sum_[:3], [3, 5, 6]) def test_sum_series_wildcards_empty_series_int_position(self): @@ -853,11 +1021,13 @@ def test_sum_series_wildcards_empty_series_int_position(self): def test_sum_series_wildcards(self): series = self._generate_series_list() [sum_] = functions.sumSeriesWithWildcards({}, series, 1) - self.assertEqual(sum_.pathExpression, - "sumSeries(collectd.test-db4.load.value," - "sumSeries(collectd.test-db3.load.value," - "sumSeries(collectd.test-db1.load.value," - "collectd.test-db2.load.value)))") + self.assertEqual( + sum_.pathExpression, + "sumSeries(collectd.test-db4.load.value," + "sumSeries(collectd.test-db3.load.value," + "sumSeries(collectd.test-db1.load.value," + "collectd.test-db2.load.value)))", + ) self.assertEqual(sum_[:3], [3, 5, 6]) def test_diff_series_empty(self): @@ -876,7 +1046,7 @@ def test_average_series_empty(self): def test_average_series(self): series = self._generate_series_list() average = functions.averageSeries({}, series)[0] - self.assertEqual(average[:3], [1.0, 5/3., 3.0]) + self.assertEqual(average[:3], [1.0, 5 / 3.0, 3.0]) def test_stddev_series_empty(self): self.assertEqual(functions.stddevSeries({}, None), []) @@ -937,44 +1107,344 @@ def test_keep_last_value(self): def test_interpolate(self): seriesList = [ - TimeSeries('collectd.test-db1.load.value', 0, 1, 1, - [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20]), - TimeSeries('collectd.test-db2.load.value', 0, 1, 1, - [None, 2, None, 4, None, 6, None, 8, None, 10, None, - 12, None, 14, None, 16, None, 18, None, 20]), - TimeSeries('collectd.test-db3.load.value', 0, 1, 1, - [1, 2, None, None, None, 6, 7, 8, 9, 10, 11, 12, 13, - 14, 15, 16, 17, None, None, None]), - TimeSeries('collectd.test-db4.load.value', 0, 1, 1, - [1, 2, 3, 4, None, 6, None, None, 9, 10, 11, None, 13, - None, None, None, None, 18, 19, 20]), - TimeSeries('collectd.test-db5.load.value', 0, 1, 1, - [1, 2, None, None, None, 6, 7, 8, 9, 10, 11, 12, 13, - 14, 15, 16, 17, 18, None, None]), - TimeSeries('collectd.test-db6.load.value', 0, 1, 1, - [None, None, None, None, 5, 6, 7, 8, 9, 10, 11, 12, 13, - 14, 15, 16, 17, 18, None, None]), + TimeSeries( + "collectd.test-db1.load.value", + 0, + 1, + 1, + [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + ], + ), + TimeSeries( + "collectd.test-db2.load.value", + 0, + 1, + 1, + [ + None, + 2, + None, + 4, + None, + 6, + None, + 8, + None, + 10, + None, + 12, + None, + 14, + None, + 16, + None, + 18, + None, + 20, + ], + ), + TimeSeries( + "collectd.test-db3.load.value", + 0, + 1, + 1, + [ + 1, + 2, + None, + None, + None, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + None, + None, + None, + ], + ), + TimeSeries( + "collectd.test-db4.load.value", + 0, + 1, + 1, + [ + 1, + 2, + 3, + 4, + None, + 6, + None, + None, + 9, + 10, + 11, + None, + 13, + None, + None, + None, + None, + 18, + 19, + 20, + ], + ), + TimeSeries( + "collectd.test-db5.load.value", + 0, + 1, + 1, + [ + 1, + 2, + None, + None, + None, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + None, + None, + ], + ), + TimeSeries( + "collectd.test-db6.load.value", + 0, + 1, + 1, + [ + None, + None, + None, + None, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + None, + None, + ], + ), ] expectedResult = [ - TimeSeries('interpolate(collectd.test-db1.load.value)', 0, 1, 1, - [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20]), - TimeSeries('interpolate(collectd.test-db2.load.value)', 0, 1, 1, - [None, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20]), - TimeSeries('interpolate(collectd.test-db3.load.value)', 0, 1, 1, - [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, - 17, None, None, None]), - TimeSeries('interpolate(collectd.test-db4.load.value)', 0, 1, 1, - [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, - 17, 18, 19, 20]), - TimeSeries('interpolate(collectd.test-db5.load.value)', 0, 1, 1, - [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, - 17, 18, None, None]), - TimeSeries('interpolate(collectd.test-db6.load.value)', 0, 1, 1, - [None, None, None, None, 5, 6, 7, 8, 9, 10, 11, 12, 13, - 14, 15, 16, 17, 18, None, None]), + TimeSeries( + "interpolate(collectd.test-db1.load.value)", + 0, + 1, + 1, + [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + ], + ), + TimeSeries( + "interpolate(collectd.test-db2.load.value)", + 0, + 1, + 1, + [ + None, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + ], + ), + TimeSeries( + "interpolate(collectd.test-db3.load.value)", + 0, + 1, + 1, + [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + None, + None, + None, + ], + ), + TimeSeries( + "interpolate(collectd.test-db4.load.value)", + 0, + 1, + 1, + [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + ], + ), + TimeSeries( + "interpolate(collectd.test-db5.load.value)", + 0, + 1, + 1, + [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + None, + None, + ], + ), + TimeSeries( + "interpolate(collectd.test-db6.load.value)", + 0, + 1, + 1, + [ + None, + None, + None, + None, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + None, + None, + ], + ), ] results = functions.interpolate({}, seriesList) self.assertEqual(results, expectedResult) @@ -1006,7 +1476,7 @@ def test_as_percent(self): def test_divide_series(self): series = self._generate_series_list() div = functions.divideSeries({}, [series[0]], [series[1]])[0] - self.assertEqual(div[:3], [0, 1/3., 0.5]) + self.assertEqual(div[:3], [0, 1 / 3.0, 0.5]) with self.assertRaises(ValueError): functions.divideSeries({}, [series[0]], [1, 2]) @@ -1045,8 +1515,13 @@ def test_exponential_moving_average_empty_series(self): def test_exponential_moving_average_returns_none(self): def gen_series_list(start=0): seriesList = [ - TimeSeries('collectd.test-db0.load.value', - start+10, start+15, 1, range(start, start+15)), + TimeSeries( + "collectd.test-db0.load.value", + start + 10, + start + 15, + 1, + range(start, start + 15), + ) ] for series in seriesList: series.pathExpression = series.name @@ -1056,8 +1531,9 @@ def gen_series_list(start=0): def mock_evaluate(reqCtx, tokens, store=None): seriesList = [ - TimeSeries('collectd.test-db0.load.value', 10, 25, 1, - [None] * 15) + TimeSeries( + "collectd.test-db0.load.value", 10, 25, 1, [None] * 15 + ) ] for series in seriesList: series.pathExpression = series.name @@ -1065,25 +1541,38 @@ def mock_evaluate(reqCtx, tokens, store=None): expectedResults = [ TimeSeries( - 'exponentialMovingAverage(collectd.test-db0.load.value,10)', - 20, 25, 1, [None, None, None, None, None]) + "exponentialMovingAverage(collectd.test-db0.load.value,10)", + 20, + 25, + 1, + [None, None, None, None, None], + ) ] - with patch('graphite_api.functions.evaluateTokens', mock_evaluate): - result = functions.exponentialMovingAverage({ - 'args': ({}, {}), - 'startTime': datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), - 'data': [], - }, series, 10) + with patch("graphite_api.functions.evaluateTokens", mock_evaluate): + result = functions.exponentialMovingAverage( + { + "args": ({}, {}), + "startTime": datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), + "data": [], + }, + series, + 10, + ) self.assertEqual(result, expectedResults) def test_exponential_moving_average_returns_half_none(self): def gen_series_list(start=0): seriesList = [ - TimeSeries('collectd.test-db0.load.value', - start+10, start+20, 1, range(0, 10)), + TimeSeries( + "collectd.test-db0.load.value", + start + 10, + start + 20, + 1, + range(0, 10), + ) ] for series in seriesList: series.pathExpression = series.name @@ -1093,8 +1582,13 @@ def gen_series_list(start=0): def mock_evaluate(reqCtx, tokens, store=None): seriesList = [ - TimeSeries('collectd.test-db0.load.value', 10, 30, 1, - [None] * 10 + list(range(0, 10))) + TimeSeries( + "collectd.test-db0.load.value", + 10, + 30, + 1, + [None] * 10 + list(range(0, 10)), + ) ] for series in seriesList: series.pathExpression = series.name @@ -1102,26 +1596,49 @@ def mock_evaluate(reqCtx, tokens, store=None): expectedResults = [ TimeSeries( - 'exponentialMovingAverage(collectd.test-db0.load.value,10)', - 20, 30, 1, [None, 0.0, 0.182, 0.512, 0.965, - 1.517, 2.15, 2.85, 3.604, 4.404]) + "exponentialMovingAverage(collectd.test-db0.load.value,10)", + 20, + 30, + 1, + [ + None, + 0.0, + 0.182, + 0.512, + 0.965, + 1.517, + 2.15, + 2.85, + 3.604, + 4.404, + ], + ) ] - with patch('graphite_api.functions.evaluateTokens', mock_evaluate): - result = functions.exponentialMovingAverage({ - 'args': ({}, {}), - 'startTime': datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), - 'data': [], - }, series, 10) + with patch("graphite_api.functions.evaluateTokens", mock_evaluate): + result = functions.exponentialMovingAverage( + { + "args": ({}, {}), + "startTime": datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), + "data": [], + }, + series, + 10, + ) self.assertEqual(result, expectedResults) def test_exponential_moving_average_returns_empty(self): def gen_series_list(start=0): seriesList = [ - TimeSeries('collectd.test-db0.load.value', - start+600, start+700, 1, range(start, start+100)), + TimeSeries( + "collectd.test-db0.load.value", + start + 600, + start + 700, + 1, + range(start, start + 100), + ) ] for series in seriesList: series.pathExpression = series.name @@ -1134,21 +1651,30 @@ def mock_evaluate(reqCtx, tokens, store=None): expectedResults = [] - with patch('graphite_api.functions.evaluateTokens', mock_evaluate): - result = functions.exponentialMovingAverage({ - 'args': ({}, {}), - 'startTime': datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), - 'data': [], - }, series, 60) + with patch("graphite_api.functions.evaluateTokens", mock_evaluate): + result = functions.exponentialMovingAverage( + { + "args": ({}, {}), + "startTime": datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), + "data": [], + }, + series, + 60, + ) self.assertEqual(result, expectedResults) def test_exponential_moving_average(self): def gen_series_list(start=0): seriesList = [ - TimeSeries('collectd.test-db0.load.value', - start+600, start+700, 1, range(start, start+100)), + TimeSeries( + "collectd.test-db0.load.value", + start + 600, + start + 700, + 1, + range(start, start + 100), + ) ] for series in seriesList: series.pathExpression = series.name @@ -1164,31 +1690,52 @@ def frange(x, y, jump): yield x x += jump - expectedResults = [[ - TimeSeries( - 'exponentialMovingAverage(collectd.test-db0.load.value,60)', - 660, 700, 1, frange(29.5, 69.5, 1)), - ], [ - TimeSeries( - 'exponentialMovingAverage(collectd.test-db0.load.value,' - '"-1min")', 660, 700, 1, frange(29.5, 69.5, 1)), - ]] - - with patch('graphite_api.functions.evaluateTokens', mock_evaluate): - result = functions.exponentialMovingAverage({ - 'args': ({}, {}), - 'startTime': datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), - 'data': [], - }, series, 60) + expectedResults = [ + [ + TimeSeries( + "exponentialMovingAverage(collectd.test-db0.load.value," + "60)", + 660, + 700, + 1, + frange(29.5, 69.5, 1), + ) + ], + [ + TimeSeries( + "exponentialMovingAverage(collectd.test-db0.load.value," + '"-1min")', + 660, + 700, + 1, + frange(29.5, 69.5, 1), + ) + ], + ] + + with patch("graphite_api.functions.evaluateTokens", mock_evaluate): + result = functions.exponentialMovingAverage( + { + "args": ({}, {}), + "startTime": datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), + "data": [], + }, + series, + 60, + ) self.assertEqual(result, expectedResults[0]) - result = functions.exponentialMovingAverage({ - 'args': ({}, {}), - 'startTime': datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), - 'data': [], - }, series, "-1min") + result = functions.exponentialMovingAverage( + { + "args": ({}, {}), + "startTime": datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), + "data": [], + }, + series, + "-1min", + ) self.assertEqual(result, expectedResults[1]) def test_moving_median_empty_series(self): @@ -1197,8 +1744,13 @@ def test_moving_median_empty_series(self): def test_moving_median_returns_none(self): def gen_series_list(start=0): seriesList = [ - TimeSeries('collectd.test-db0.load.value', - start+10, start+15, 1, range(start, start+15)), + TimeSeries( + "collectd.test-db0.load.value", + start + 10, + start + 15, + 1, + range(start, start + 15), + ) ] for series in seriesList: series.pathExpression = series.name @@ -1208,33 +1760,48 @@ def gen_series_list(start=0): def mock_evaluate(reqCtx, tokens, store=None): seriesList = [ - TimeSeries('collectd.test-db0.load.value', 10, 25, 1, - [None] * 15) + TimeSeries( + "collectd.test-db0.load.value", 10, 25, 1, [None] * 15 + ) ] for series in seriesList: series.pathExpression = series.name return seriesList expectedResults = [ - TimeSeries('movingMedian(collectd.test-db0.load.value,10)', - 20, 25, 1, [None, None, None, None, None]) + TimeSeries( + "movingMedian(collectd.test-db0.load.value,10)", + 20, + 25, + 1, + [None, None, None, None, None], + ) ] - with patch('graphite_api.functions.evaluateTokens', mock_evaluate): - result = functions.movingMedian({ - 'args': ({}, {}), - 'startTime': datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), - 'data': [], - }, series, 10) + with patch("graphite_api.functions.evaluateTokens", mock_evaluate): + result = functions.movingMedian( + { + "args": ({}, {}), + "startTime": datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), + "data": [], + }, + series, + 10, + ) self.assertEqual(result, expectedResults) def test_moving_median_returns_half_none(self): def gen_series_list(start=0): seriesList = [ - TimeSeries('collectd.test-db0.load.value', - start+10, start+20, 1, range(0, 10)), + TimeSeries( + "collectd.test-db0.load.value", + start + 10, + start + 20, + 1, + range(0, 10), + ) ] for series in seriesList: series.pathExpression = series.name @@ -1244,33 +1811,52 @@ def gen_series_list(start=0): def mock_evaluate(reqCtx, tokens, store=None): seriesList = [ - TimeSeries('collectd.test-db0.load.value', 10, 30, 1, - [None] * 10 + list(range(0, 10))) + TimeSeries( + "collectd.test-db0.load.value", + 10, + 30, + 1, + [None] * 10 + list(range(0, 10)), + ) ] for series in seriesList: series.pathExpression = series.name return seriesList expectedResults = [ - TimeSeries('movingMedian(collectd.test-db0.load.value,10)', - 20, 30, 1, [None, 0, 1, 1, 2, 2, 3, 3, 4, 4]) + TimeSeries( + "movingMedian(collectd.test-db0.load.value,10)", + 20, + 30, + 1, + [None, 0, 1, 1, 2, 2, 3, 3, 4, 4], + ) ] - with patch('graphite_api.functions.evaluateTokens', mock_evaluate): - result = functions.movingMedian({ - 'args': ({}, {}), - 'startTime': datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), - 'data': [], - }, series, 10) + with patch("graphite_api.functions.evaluateTokens", mock_evaluate): + result = functions.movingMedian( + { + "args": ({}, {}), + "startTime": datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), + "data": [], + }, + series, + 10, + ) self.assertEqual(result, expectedResults) def test_moving_median_returns_empty(self): def gen_series_list(start=0): seriesList = [ - TimeSeries('collectd.test-db0.load.value', - start+600, start+700, 1, range(start, start+100)), + TimeSeries( + "collectd.test-db0.load.value", + start + 600, + start + 700, + 1, + range(start, start + 100), + ) ] for series in seriesList: series.pathExpression = series.name @@ -1283,21 +1869,30 @@ def mock_evaluate(reqCtx, tokens, store=None): expectedResults = [] - with patch('graphite_api.functions.evaluateTokens', mock_evaluate): - result = functions.movingMedian({ - 'args': ({}, {}), - 'startTime': datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), - 'data': [], - }, series, 60) + with patch("graphite_api.functions.evaluateTokens", mock_evaluate): + result = functions.movingMedian( + { + "args": ({}, {}), + "startTime": datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), + "data": [], + }, + series, + 60, + ) self.assertEqual(result, expectedResults) def test_moving_median(self): def gen_series_list(start=0): seriesList = [ - TimeSeries('collectd.test-db0.load.value', - start+600, start+700, 1, range(start, start+100)), + TimeSeries( + "collectd.test-db0.load.value", + start + 600, + start + 700, + 1, + range(start, start + 100), + ) ] for series in seriesList: series.pathExpression = series.name @@ -1308,29 +1903,50 @@ def gen_series_list(start=0): def mock_evaluate(reqCtx, tokens, store=None): return gen_series_list() - expectedResults = [[ - TimeSeries('movingMedian(collectd.test-db0.load.value,60)', - 660, 700, 1, range(30, 70)), - ], [ - TimeSeries('movingMedian(collectd.test-db0.load.value,"-1min")', - 660, 700, 1, range(30, 70)), - ]] - - with patch('graphite_api.functions.evaluateTokens', mock_evaluate): - result = functions.movingMedian({ - 'args': ({}, {}), - 'startTime': datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), - 'data': [], - }, series, 60) + expectedResults = [ + [ + TimeSeries( + "movingMedian(collectd.test-db0.load.value,60)", + 660, + 700, + 1, + range(30, 70), + ) + ], + [ + TimeSeries( + 'movingMedian(collectd.test-db0.load.value,"-1min")', + 660, + 700, + 1, + range(30, 70), + ) + ], + ] + + with patch("graphite_api.functions.evaluateTokens", mock_evaluate): + result = functions.movingMedian( + { + "args": ({}, {}), + "startTime": datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), + "data": [], + }, + series, + 60, + ) self.assertEqual(result, expectedResults[0]) - result = functions.movingMedian({ - 'args': ({}, {}), - 'startTime': datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), - 'data': [], - }, series, "-1min") + result = functions.movingMedian( + { + "args": ({}, {}), + "startTime": datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), + "data": [], + }, + series, + "-1min", + ) self.assertEqual(result, expectedResults[1]) def test_pow_series_empty(self): @@ -1339,16 +1955,42 @@ def test_pow_series_empty(self): def test_pow_series(self): seriesList = [ - TimeSeries('collectd.test-db1.load.value', 0, 1, 1, - [1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 312.1]), - TimeSeries('collectd.test-db2.load.value', 0, 1, 1, - [1, 3, 5, 7, None, 6, 4, 8, 0, 10, 234.2]), + TimeSeries( + "collectd.test-db1.load.value", + 0, + 1, + 1, + [1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 312.1], + ), + TimeSeries( + "collectd.test-db2.load.value", + 0, + 1, + 1, + [1, 3, 5, 7, None, 6, 4, 8, 0, 10, 234.2], + ), ] expectedResult = [ - TimeSeries('powSeries(collectd.test-db1.load.value,' - 'collectd.test-db2.load.value)', 0, 1, 1, - [1.0, 8.0, 243.0, 16384.0, None, 46656.0, - 2401.0, 16777216.0, 1.0, 0.0, None]), + TimeSeries( + "powSeries(collectd.test-db1.load.value," + "collectd.test-db2.load.value)", + 0, + 1, + 1, + [ + 1.0, + 8.0, + 243.0, + 16384.0, + None, + 46656.0, + 2401.0, + 16777216.0, + 1.0, + 0.0, + None, + ], + ) ] result = functions.powSeries({}, seriesList) @@ -1357,13 +1999,14 @@ def test_pow_series(self): def test_square_root(self): series = self._generate_series_list() square_root = functions.squareRoot({}, series)[0] - self.assertEqual(square_root[:5], - [0.0, 1.0, math.sqrt(2), math.sqrt(3), 2.0]) + self.assertEqual( + square_root[:5], [0.0, 1.0, math.sqrt(2), math.sqrt(3), 2.0] + ) def test_invert(self): series = self._generate_series_list() invert = functions.invert({}, series)[0] - self.assertEqual(invert[:5], [None, 1, 1/2., 1/3., 1/4.]) + self.assertEqual(invert[:5], [None, 1, 1 / 2.0, 1 / 3.0, 1 / 4.0]) def test_scale_to_seconds(self): series = self._generate_series_list() @@ -1382,7 +2025,8 @@ def test_offset(self): def test_offset_to_zero(self): series = self._generate_series_list( - config=[[None] + list(range(10, 110))]) + config=[[None] + list(range(10, 110))] + ) offset = functions.offsetToZero({}, series)[0] self.assertEqual(offset[:3], [None, 0, 1]) @@ -1392,8 +2036,13 @@ def test_moving_average_empty_series(self): def test_moving_average_returns_none(self): def gen_series_list(start=0): seriesList = [ - TimeSeries('collectd.test-db0.load.value', - start+10, start+15, 1, range(start, start+15)), + TimeSeries( + "collectd.test-db0.load.value", + start + 10, + start + 15, + 1, + range(start, start + 15), + ) ] for series in seriesList: series.pathExpression = series.name @@ -1403,33 +2052,48 @@ def gen_series_list(start=0): def mock_evaluate(reqCtx, tokens, store=None): seriesList = [ - TimeSeries('collectd.test-db0.load.value', 10, 25, 1, - [None] * 15) + TimeSeries( + "collectd.test-db0.load.value", 10, 25, 1, [None] * 15 + ) ] for series in seriesList: series.pathExpression = series.name return seriesList expectedResults = [ - TimeSeries('movingAverage(collectd.test-db0.load.value,10)', - 20, 25, 1, [None, None, None, None, None]) + TimeSeries( + "movingAverage(collectd.test-db0.load.value,10)", + 20, + 25, + 1, + [None, None, None, None, None], + ) ] - with patch('graphite_api.functions.evaluateTokens', mock_evaluate): - result = functions.movingAverage({ - 'args': ({}, {}), - 'startTime': datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), - 'data': [], - }, series, 10) + with patch("graphite_api.functions.evaluateTokens", mock_evaluate): + result = functions.movingAverage( + { + "args": ({}, {}), + "startTime": datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), + "data": [], + }, + series, + 10, + ) self.assertEqual(result, expectedResults) def test_moving_average_returns_half_none(self): def gen_series_list(start=0): seriesList = [ - TimeSeries('collectd.test-db0.load.value', - start+10, start+20, 1, range(0, 10)), + TimeSeries( + "collectd.test-db0.load.value", + start + 10, + start + 20, + 1, + range(0, 10), + ) ] for series in seriesList: series.pathExpression = series.name @@ -1439,34 +2103,52 @@ def gen_series_list(start=0): def mock_evaluate(reqCtx, tokens, store=None): seriesList = [ - TimeSeries('collectd.test-db0.load.value', 10, 30, 1, - [None] * 10 + list(range(0, 10))) + TimeSeries( + "collectd.test-db0.load.value", + 10, + 30, + 1, + [None] * 10 + list(range(0, 10)), + ) ] for series in seriesList: series.pathExpression = series.name return seriesList expectedResults = [ - TimeSeries('movingAverage(collectd.test-db0.load.value,10)', - 20, 30, 1, [None, 0.0, 0.5, 1.0, 1.5, - 2.0, 2.5, 3.0, 3.5, 4.0]) + TimeSeries( + "movingAverage(collectd.test-db0.load.value,10)", + 20, + 30, + 1, + [None, 0.0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0], + ) ] - with patch('graphite_api.functions.evaluateTokens', mock_evaluate): - result = functions.movingAverage({ - 'args': ({}, {}), - 'startTime': datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), - 'data': [], - }, series, 10) + with patch("graphite_api.functions.evaluateTokens", mock_evaluate): + result = functions.movingAverage( + { + "args": ({}, {}), + "startTime": datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), + "data": [], + }, + series, + 10, + ) self.assertEqual(result, expectedResults) def test_moving_average_returns_empty(self): def gen_series_list(start=0): seriesList = [ - TimeSeries('collectd.test-db0.load.value', - start+600, start+700, 1, range(start, start+100)), + TimeSeries( + "collectd.test-db0.load.value", + start + 600, + start + 700, + 1, + range(start, start + 100), + ) ] for series in seriesList: series.pathExpression = series.name @@ -1479,21 +2161,30 @@ def mock_evaluate(reqCtx, tokens, store=None): expectedResults = [] - with patch('graphite_api.functions.evaluateTokens', mock_evaluate): - result = functions.movingAverage({ - 'args': ({}, {}), - 'startTime': datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), - 'data': [], - }, series, 60) + with patch("graphite_api.functions.evaluateTokens", mock_evaluate): + result = functions.movingAverage( + { + "args": ({}, {}), + "startTime": datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), + "data": [], + }, + series, + 60, + ) self.assertEqual(result, expectedResults) def test_moving_average(self): def gen_series_list(start=0): seriesList = [ - TimeSeries('collectd.test-db0.load.value', - start+600, start+700, 1, range(start, start+100)), + TimeSeries( + "collectd.test-db0.load.value", + start + 600, + start + 700, + 1, + range(start, start + 100), + ) ] for series in seriesList: series.pathExpression = series.name @@ -1509,29 +2200,50 @@ def frange(x, y, jump): yield x x += jump - expectedResults = [[ - TimeSeries('movingAverage(collectd.test-db0.load.value,60)', - 660, 700, 1, frange(29.5, 69.5, 1)), - ], [ - TimeSeries('movingAverage(collectd.test-db0.load.value,"-1min")', - 660, 700, 1, frange(29.5, 69.5, 1)), - ]] - - with patch('graphite_api.functions.evaluateTokens', mock_evaluate): - result = functions.movingAverage({ - 'args': ({}, {}), - 'startTime': datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), - 'data': [], - }, series, 60) + expectedResults = [ + [ + TimeSeries( + "movingAverage(collectd.test-db0.load.value,60)", + 660, + 700, + 1, + frange(29.5, 69.5, 1), + ) + ], + [ + TimeSeries( + 'movingAverage(collectd.test-db0.load.value,"-1min")', + 660, + 700, + 1, + frange(29.5, 69.5, 1), + ) + ], + ] + + with patch("graphite_api.functions.evaluateTokens", mock_evaluate): + result = functions.movingAverage( + { + "args": ({}, {}), + "startTime": datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), + "data": [], + }, + series, + 60, + ) self.assertEqual(result, expectedResults[0]) - result = functions.movingAverage({ - 'args': ({}, {}), - 'startTime': datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), - 'data': [], - }, series, "-1min") + result = functions.movingAverage( + { + "args": ({}, {}), + "startTime": datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), + "data": [], + }, + series, + "-1min", + ) self.assertEqual(result, expectedResults[1]) def test_moving_min_empty_series(self): @@ -1540,8 +2252,13 @@ def test_moving_min_empty_series(self): def test_moving_min_returns_none(self): def gen_series_list(start=0): seriesList = [ - TimeSeries('collectd.test-db0.load.value', - start+10, start+15, 1, range(start, start+15)), + TimeSeries( + "collectd.test-db0.load.value", + start + 10, + start + 15, + 1, + range(start, start + 15), + ) ] for series in seriesList: series.pathExpression = series.name @@ -1551,33 +2268,48 @@ def gen_series_list(start=0): def mock_evaluate(reqCtx, tokens, store=None): seriesList = [ - TimeSeries('collectd.test-db0.load.value', 10, 25, 1, - [None] * 15) + TimeSeries( + "collectd.test-db0.load.value", 10, 25, 1, [None] * 15 + ) ] for series in seriesList: series.pathExpression = series.name return seriesList expectedResults = [ - TimeSeries('movingMin(collectd.test-db0.load.value,10)', - 20, 25, 1, [None, None, None, None, None]) + TimeSeries( + "movingMin(collectd.test-db0.load.value,10)", + 20, + 25, + 1, + [None, None, None, None, None], + ) ] - with patch('graphite_api.functions.evaluateTokens', mock_evaluate): - result = functions.movingMin({ - 'args': ({}, {}), - 'startTime': datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), - 'data': [], - }, series, 10) + with patch("graphite_api.functions.evaluateTokens", mock_evaluate): + result = functions.movingMin( + { + "args": ({}, {}), + "startTime": datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), + "data": [], + }, + series, + 10, + ) self.assertEqual(result, expectedResults) def test_moving_min_returns_half_none(self): def gen_series_list(start=0): seriesList = [ - TimeSeries('collectd.test-db0.load.value', - start+10, start+20, 1, range(0, 10)), + TimeSeries( + "collectd.test-db0.load.value", + start + 10, + start + 20, + 1, + range(0, 10), + ) ] for series in seriesList: series.pathExpression = series.name @@ -1587,33 +2319,52 @@ def gen_series_list(start=0): def mock_evaluate(reqCtx, tokens, store=None): seriesList = [ - TimeSeries('collectd.test-db0.load.value', 10, 30, 1, - [None] * 10 + list(range(0, 10))) + TimeSeries( + "collectd.test-db0.load.value", + 10, + 30, + 1, + [None] * 10 + list(range(0, 10)), + ) ] for series in seriesList: series.pathExpression = series.name return seriesList expectedResults = [ - TimeSeries('movingMin(collectd.test-db0.load.value,10)', - 20, 30, 1, [None, 0, 0, 0, 0, 0, 0, 0, 0, 0]) + TimeSeries( + "movingMin(collectd.test-db0.load.value,10)", + 20, + 30, + 1, + [None, 0, 0, 0, 0, 0, 0, 0, 0, 0], + ) ] - with patch('graphite_api.functions.evaluateTokens', mock_evaluate): - result = functions.movingMin({ - 'args': ({}, {}), - 'startTime': datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), - 'data': [], - }, series, 10) + with patch("graphite_api.functions.evaluateTokens", mock_evaluate): + result = functions.movingMin( + { + "args": ({}, {}), + "startTime": datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), + "data": [], + }, + series, + 10, + ) self.assertEqual(result, expectedResults) def test_moving_min_returns_empty(self): def gen_series_list(start=0): seriesList = [ - TimeSeries('collectd.test-db0.load.value', - start+600, start+700, 1, range(start, start+100)), + TimeSeries( + "collectd.test-db0.load.value", + start + 600, + start + 700, + 1, + range(start, start + 100), + ) ] for series in seriesList: series.pathExpression = series.name @@ -1626,21 +2377,30 @@ def mock_evaluate(reqCtx, tokens, store=None): expectedResults = [] - with patch('graphite_api.functions.evaluateTokens', mock_evaluate): - result = functions.movingMin({ - 'args': ({}, {}), - 'startTime': datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), - 'data': [], - }, series, 60) + with patch("graphite_api.functions.evaluateTokens", mock_evaluate): + result = functions.movingMin( + { + "args": ({}, {}), + "startTime": datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), + "data": [], + }, + series, + 60, + ) self.assertEqual(result, expectedResults) def test_moving_min(self): def gen_series_list(start=0): seriesList = [ - TimeSeries('collectd.test-db0.load.value', - start+600, start+700, 1, range(start, start+100)), + TimeSeries( + "collectd.test-db0.load.value", + start + 600, + start + 700, + 1, + range(start, start + 100), + ) ] for series in seriesList: series.pathExpression = series.name @@ -1651,29 +2411,50 @@ def gen_series_list(start=0): def mock_evaluate(reqCtx, tokens, store=None): return gen_series_list() - expectedResults = [[ - TimeSeries('movingMin(collectd.test-db0.load.value,60)', - 660, 700, 1, range(0, 40)), - ], [ - TimeSeries('movingMin(collectd.test-db0.load.value,"-1min")', - 660, 700, 1, range(0, 40)), - ]] - - with patch('graphite_api.functions.evaluateTokens', mock_evaluate): - result = functions.movingMin({ - 'args': ({}, {}), - 'startTime': datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), - 'data': [], - }, series, 60) + expectedResults = [ + [ + TimeSeries( + "movingMin(collectd.test-db0.load.value,60)", + 660, + 700, + 1, + range(0, 40), + ) + ], + [ + TimeSeries( + 'movingMin(collectd.test-db0.load.value,"-1min")', + 660, + 700, + 1, + range(0, 40), + ) + ], + ] + + with patch("graphite_api.functions.evaluateTokens", mock_evaluate): + result = functions.movingMin( + { + "args": ({}, {}), + "startTime": datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), + "data": [], + }, + series, + 60, + ) self.assertEqual(result, expectedResults[0]) - result = functions.movingMin({ - 'args': ({}, {}), - 'startTime': datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), - 'data': [], - }, series, "-1min") + result = functions.movingMin( + { + "args": ({}, {}), + "startTime": datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), + "data": [], + }, + series, + "-1min", + ) self.assertEqual(result, expectedResults[1]) def test_moving_max_empty_series(self): @@ -1682,8 +2463,13 @@ def test_moving_max_empty_series(self): def test_moving_max_returns_none(self): def gen_series_list(start=0): seriesList = [ - TimeSeries('collectd.test-db0.load.value', - start+10, start+15, 1, range(start, start+15)), + TimeSeries( + "collectd.test-db0.load.value", + start + 10, + start + 15, + 1, + range(start, start + 15), + ) ] for series in seriesList: series.pathExpression = series.name @@ -1693,33 +2479,48 @@ def gen_series_list(start=0): def mock_evaluate(reqCtx, tokens, store=None): seriesList = [ - TimeSeries('collectd.test-db0.load.value', 10, 25, 1, - [None] * 15) + TimeSeries( + "collectd.test-db0.load.value", 10, 25, 1, [None] * 15 + ) ] for series in seriesList: series.pathExpression = series.name return seriesList expectedResults = [ - TimeSeries('movingMax(collectd.test-db0.load.value,10)', - 20, 25, 1, [None, None, None, None, None]) + TimeSeries( + "movingMax(collectd.test-db0.load.value,10)", + 20, + 25, + 1, + [None, None, None, None, None], + ) ] - with patch('graphite_api.functions.evaluateTokens', mock_evaluate): - result = functions.movingMax({ - 'args': ({}, {}), - 'startTime': datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), - 'data': [], - }, series, 10) + with patch("graphite_api.functions.evaluateTokens", mock_evaluate): + result = functions.movingMax( + { + "args": ({}, {}), + "startTime": datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), + "data": [], + }, + series, + 10, + ) self.assertEqual(result, expectedResults) def test_moving_max_returns_half_none(self): def gen_series_list(start=0): seriesList = [ - TimeSeries('collectd.test-db0.load.value', - start+10, start+20, 1, range(0, 10)), + TimeSeries( + "collectd.test-db0.load.value", + start + 10, + start + 20, + 1, + range(0, 10), + ) ] for series in seriesList: series.pathExpression = series.name @@ -1729,33 +2530,52 @@ def gen_series_list(start=0): def mock_evaluate(reqCtx, tokens, store=None): seriesList = [ - TimeSeries('collectd.test-db0.load.value', 10, 30, 1, - [None] * 10 + list(range(0, 10))) + TimeSeries( + "collectd.test-db0.load.value", + 10, + 30, + 1, + [None] * 10 + list(range(0, 10)), + ) ] for series in seriesList: series.pathExpression = series.name return seriesList expectedResults = [ - TimeSeries('movingMax(collectd.test-db0.load.value,10)', - 20, 30, 1, [None, 0, 1, 2, 3, 4, 5, 6, 7, 8]) + TimeSeries( + "movingMax(collectd.test-db0.load.value,10)", + 20, + 30, + 1, + [None, 0, 1, 2, 3, 4, 5, 6, 7, 8], + ) ] - with patch('graphite_api.functions.evaluateTokens', mock_evaluate): - result = functions.movingMax({ - 'args': ({}, {}), - 'startTime': datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), - 'data': [], - }, series, 10) + with patch("graphite_api.functions.evaluateTokens", mock_evaluate): + result = functions.movingMax( + { + "args": ({}, {}), + "startTime": datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), + "data": [], + }, + series, + 10, + ) self.assertEqual(result, expectedResults) def test_moving_max_returns_empty(self): def gen_series_list(start=0): seriesList = [ - TimeSeries('collectd.test-db0.load.value', - start+600, start+700, 1, range(start, start+100)), + TimeSeries( + "collectd.test-db0.load.value", + start + 600, + start + 700, + 1, + range(start, start + 100), + ) ] for series in seriesList: series.pathExpression = series.name @@ -1768,21 +2588,30 @@ def mock_evaluate(reqCtx, tokens, store=None): expectedResults = [] - with patch('graphite_api.functions.evaluateTokens', mock_evaluate): - result = functions.movingMax({ - 'args': ({}, {}), - 'startTime': datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), - 'data': [], - }, series, 60) + with patch("graphite_api.functions.evaluateTokens", mock_evaluate): + result = functions.movingMax( + { + "args": ({}, {}), + "startTime": datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), + "data": [], + }, + series, + 60, + ) self.assertEqual(result, expectedResults) def test_moving_max(self): def gen_series_list(start=0): seriesList = [ - TimeSeries('collectd.test-db0.load.value', - start+600, start+700, 1, range(start, start+100)), + TimeSeries( + "collectd.test-db0.load.value", + start + 600, + start + 700, + 1, + range(start, start + 100), + ) ] for series in seriesList: series.pathExpression = series.name @@ -1793,29 +2622,50 @@ def gen_series_list(start=0): def mock_evaluate(reqCtx, tokens, store=None): return gen_series_list() - expectedResults = [[ - TimeSeries('movingMax(collectd.test-db0.load.value,60)', - 660, 700, 1, range(59, 99)), - ], [ - TimeSeries('movingMax(collectd.test-db0.load.value,"-1min")', - 660, 700, 1, range(59, 99)), - ]] - - with patch('graphite_api.functions.evaluateTokens', mock_evaluate): - result = functions.movingMax({ - 'args': ({}, {}), - 'startTime': datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), - 'data': [], - }, series, 60) + expectedResults = [ + [ + TimeSeries( + "movingMax(collectd.test-db0.load.value,60)", + 660, + 700, + 1, + range(59, 99), + ) + ], + [ + TimeSeries( + 'movingMax(collectd.test-db0.load.value,"-1min")', + 660, + 700, + 1, + range(59, 99), + ) + ], + ] + + with patch("graphite_api.functions.evaluateTokens", mock_evaluate): + result = functions.movingMax( + { + "args": ({}, {}), + "startTime": datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), + "data": [], + }, + series, + 60, + ) self.assertEqual(result, expectedResults[0]) - result = functions.movingMax({ - 'args': ({}, {}), - 'startTime': datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), - 'data': [], - }, series, "-1min") + result = functions.movingMax( + { + "args": ({}, {}), + "startTime": datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), + "data": [], + }, + series, + "-1min", + ) self.assertEqual(result, expectedResults[1]) def test_moving_sum_empty_series(self): @@ -1824,8 +2674,13 @@ def test_moving_sum_empty_series(self): def test_moving_sum_returns_none(self): def gen_series_list(start=0): seriesList = [ - TimeSeries('collectd.test-db0.load.value', - start+10, start+15, 1, range(start, start+15)), + TimeSeries( + "collectd.test-db0.load.value", + start + 10, + start + 15, + 1, + range(start, start + 15), + ) ] for series in seriesList: series.pathExpression = series.name @@ -1835,33 +2690,48 @@ def gen_series_list(start=0): def mock_evaluate(reqCtx, tokens, store=None): seriesList = [ - TimeSeries('collectd.test-db0.load.value', 10, 25, 1, - [None] * 15) + TimeSeries( + "collectd.test-db0.load.value", 10, 25, 1, [None] * 15 + ) ] for series in seriesList: series.pathExpression = series.name return seriesList expectedResults = [ - TimeSeries('movingSum(collectd.test-db0.load.value,10)', - 20, 25, 1, [None, None, None, None, None]) + TimeSeries( + "movingSum(collectd.test-db0.load.value,10)", + 20, + 25, + 1, + [None, None, None, None, None], + ) ] - with patch('graphite_api.functions.evaluateTokens', mock_evaluate): - result = functions.movingSum({ - 'args': ({}, {}), - 'startTime': datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), - 'data': [], - }, series, 10) + with patch("graphite_api.functions.evaluateTokens", mock_evaluate): + result = functions.movingSum( + { + "args": ({}, {}), + "startTime": datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), + "data": [], + }, + series, + 10, + ) self.assertEqual(result, expectedResults) def test_moving_sum_returns_half_none(self): def gen_series_list(start=0): seriesList = [ - TimeSeries('collectd.test-db0.load.value', - start+10, start+20, 1, range(0, 10)), + TimeSeries( + "collectd.test-db0.load.value", + start + 10, + start + 20, + 1, + range(0, 10), + ) ] for series in seriesList: series.pathExpression = series.name @@ -1871,33 +2741,52 @@ def gen_series_list(start=0): def mock_evaluate(reqCtx, tokens, store=None): seriesList = [ - TimeSeries('collectd.test-db0.load.value', 10, 30, 1, - [None] * 10 + list(range(0, 10))) + TimeSeries( + "collectd.test-db0.load.value", + 10, + 30, + 1, + [None] * 10 + list(range(0, 10)), + ) ] for series in seriesList: series.pathExpression = series.name return seriesList expectedResults = [ - TimeSeries('movingSum(collectd.test-db0.load.value,10)', - 20, 30, 1, [None, 0, 1, 3, 6, 10, 15, 21, 28, 36]) + TimeSeries( + "movingSum(collectd.test-db0.load.value,10)", + 20, + 30, + 1, + [None, 0, 1, 3, 6, 10, 15, 21, 28, 36], + ) ] - with patch('graphite_api.functions.evaluateTokens', mock_evaluate): - result = functions.movingSum({ - 'args': ({}, {}), - 'startTime': datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), - 'data': [], - }, series, 10) + with patch("graphite_api.functions.evaluateTokens", mock_evaluate): + result = functions.movingSum( + { + "args": ({}, {}), + "startTime": datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), + "data": [], + }, + series, + 10, + ) self.assertEqual(result, expectedResults) def test_moving_sum_returns_empty(self): def gen_series_list(start=0): seriesList = [ - TimeSeries('collectd.test-db0.load.value', - start+600, start+700, 1, range(start, start+100)), + TimeSeries( + "collectd.test-db0.load.value", + start + 600, + start + 700, + 1, + range(start, start + 100), + ) ] for series in seriesList: series.pathExpression = series.name @@ -1910,21 +2799,30 @@ def mock_evaluate(reqCtx, tokens, store=None): expectedResults = [] - with patch('graphite_api.functions.evaluateTokens', mock_evaluate): - result = functions.movingSum({ - 'args': ({}, {}), - 'startTime': datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), - 'data': [], - }, series, 60) + with patch("graphite_api.functions.evaluateTokens", mock_evaluate): + result = functions.movingSum( + { + "args": ({}, {}), + "startTime": datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), + "data": [], + }, + series, + 60, + ) self.assertEqual(result, expectedResults) def test_moving_sum(self): def gen_series_list(start=0): seriesList = [ - TimeSeries('collectd.test-db0.load.value', - start+600, start+700, 1, range(start, start+100)), + TimeSeries( + "collectd.test-db0.load.value", + start + 600, + start + 700, + 1, + range(start, start + 100), + ) ] for series in seriesList: series.pathExpression = series.name @@ -1935,29 +2833,50 @@ def gen_series_list(start=0): def mock_evaluate(reqCtx, tokens, store=None): return gen_series_list() - expectedResults = [[ - TimeSeries('movingSum(collectd.test-db0.load.value,60)', - 660, 700, 1, range(1770, 4170, 60)), - ], [ - TimeSeries('movingSum(collectd.test-db0.load.value,"-1min")', - 660, 700, 1, range(1770, 4170, 60)), - ]] - - with patch('graphite_api.functions.evaluateTokens', mock_evaluate): - result = functions.movingSum({ - 'args': ({}, {}), - 'startTime': datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), - 'data': [], - }, series, 60) + expectedResults = [ + [ + TimeSeries( + "movingSum(collectd.test-db0.load.value,60)", + 660, + 700, + 1, + range(1770, 4170, 60), + ) + ], + [ + TimeSeries( + 'movingSum(collectd.test-db0.load.value,"-1min")', + 660, + 700, + 1, + range(1770, 4170, 60), + ) + ], + ] + + with patch("graphite_api.functions.evaluateTokens", mock_evaluate): + result = functions.movingSum( + { + "args": ({}, {}), + "startTime": datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), + "data": [], + }, + series, + 60, + ) self.assertEqual(result, expectedResults[0]) - result = functions.movingSum({ - 'args': ({}, {}), - 'startTime': datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), - 'data': [], - }, series, "-1min") + result = functions.movingSum( + { + "args": ({}, {}), + "startTime": datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), + "data": [], + }, + series, + "-1min", + ) self.assertEqual(result, expectedResults[1]) def test_cumulative(self): @@ -1969,13 +2888,13 @@ def test_cumulative(self): def consolidate_by(self): series = self._generate_series_list(config=[range(100)]) series[0].consolidate(2) - min_ = functions.consolidateBy({}, series, 'min') + min_ = functions.consolidateBy({}, series, "min") self.assertEqual(list(min_)[:3], [0, 2, 4]) - max_ = functions.consolidateBy({}, series, 'max') + max_ = functions.consolidateBy({}, series, "max") self.assertEqual(list(max_)[:3], [1, 3, 5]) - avg_ = functions.consolidateBy({}, series, 'average') + avg_ = functions.consolidateBy({}, series, "average") self.assertEqual(list(avg_)[:3], [0.5, 2.3, 4.5]) def test_derivative(self): @@ -1985,13 +2904,23 @@ def test_derivative(self): def test_delay(self): seriesList = [ - TimeSeries('collectd.test-db1.load.value', 0, 1, 1, - [range(18)] + [None, None]) + TimeSeries( + "collectd.test-db1.load.value", + 0, + 1, + 1, + [range(18)] + [None, None], + ) ] delay = 2 expected = [ - TimeSeries('delay(collectd.test-db1.load.value,2)', 0, 1, 1, - [None, None] + [range(18)]), + TimeSeries( + "delay(collectd.test-db1.load.value,2)", + 0, + 1, + 1, + [None, None] + [range(18)], + ) ] result = functions.delay({}, seriesList, delay) self.assertEqual(len(expected), len(result)) @@ -2010,63 +2939,81 @@ def test_per_second(self): def test_per_second_nones(self): seriesList = [ - TimeSeries('test', 0, 600, 60, - [0, 60, None, 180, None, 300, None, 420, None, 540]) + TimeSeries( + "test", + 0, + 600, + 60, + [0, 60, None, 180, None, 300, None, 420, None, 540], + ) ] expected = [ - TimeSeries('perSecond(test)', 0, 600, 60, - [None, 1, None, 1, None, 1, None, 1, None, 1]) + TimeSeries( + "perSecond(test)", + 0, + 600, + 60, + [None, 1, None, 1, None, 1, None, 1, None, 1], + ) ] result = functions.perSecond({}, seriesList) self.assertEqual(expected, result) def test_integral(self): series = self._generate_series_list( - config=[list(range(1, 10)) * 9 + [None] * 10]) + config=[list(range(1, 10)) * 9 + [None] * 10] + ) integral = functions.integral({}, series)[0] self.assertEqual(integral[:3], [1, 3, 6]) self.assertEqual(integral[-11:], [405] + [None] * 10) def test_integral_by_interval(self): series = self._generate_series_list( - config=[list(range(1, 10)) * 9 + [None] * 10]) - integral = functions.integralByInterval({ - 'startTime': parseATTime('-101s') - }, series, '100s')[0] + config=[list(range(1, 10)) * 9 + [None] * 10] + ) + integral = functions.integralByInterval( + {"startTime": parseATTime("-101s")}, series, "100s" + )[0] self.assertEqual(integral[:3], [1, 3, 6]) self.assertEqual(integral[-11:], [405] * 11) def test_non_negative_derivative(self): series = self._generate_series_list(config=[list(range(10)) * 10]) der = functions.nonNegativeDerivative({}, series)[0] - self.assertEqual(list(der), - [1 if i % 10 else None for i in range(100)]) + self.assertEqual( + list(der), [1 if i % 10 else None for i in range(100)] + ) series = self._generate_series_list( - config=[list(reversed(range(10))) * 10]) + config=[list(reversed(range(10))) * 10] + ) der = functions.nonNegativeDerivative({}, series, maxValue=10)[0] - self.assertEqual(list(der), - [None] + [10 if i % 10 else 9 for i in range(1, 100)]) + self.assertEqual( + list(der), [None] + [10 if i % 10 else 9 for i in range(1, 100)] + ) def test_stacked(self): series = self._generate_series_list( - config=[[None] + list(range(99)), range(50, 150)]) + config=[[None] + list(range(99)), range(50, 150)] + ) stacked = functions.stacked({}, series)[1] self.assertEqual(stacked[:3], [50, 51, 53]) - stacked = functions.stacked({'totalStack': {}}, series)[1] + stacked = functions.stacked({"totalStack": {}}, series)[1] self.assertEqual(stacked[:3], [50, 51, 53]) - self.assertEqual(stacked.name, 'stacked(collectd.test-db2.load.value)') + self.assertEqual( + stacked.name, "stacked(collectd.test-db2.load.value)" + ) - stacked = functions.stacked({}, series, 'tx')[1] + stacked = functions.stacked({}, series, "tx")[1] self.assertEqual(stacked[:3], [50, 51, 53]) self.assertEqual(stacked.name, series[1].name) def test_area_between(self): series = self._generate_series_list() lower, upper = functions.areaBetween({}, series[0], series[1]) - self.assertEqual(lower.options, {'stacked': True, 'invisible': True}) - self.assertEqual(upper.options, {'stacked': True}) + self.assertEqual(lower.options, {"stacked": True, "invisible": True}) + self.assertEqual(upper.options, {"stacked": True}) def test_cactistyle(self): series = self._generate_series_list() @@ -2074,68 +3021,74 @@ def test_cactistyle(self): self.assertEqual( cacti[0].name, "collectd.test-db1.load.value Current:100.00 Max:100.00 " - "Min:0.00 ") + "Min:0.00 ", + ) series = self._generate_series_list() - cacti = functions.cactiStyle({}, series, 'si') + cacti = functions.cactiStyle({}, series, "si") self.assertEqual( cacti[0].name, "collectd.test-db1.load.value Current:100.00 Max:100.00 " - "Min:0.00 ") + "Min:0.00 ", + ) series = self._generate_series_list() - cacti = functions.cactiStyle({}, series, units='b') + cacti = functions.cactiStyle({}, series, units="b") self.assertEqual( cacti[0].name, "collectd.test-db1.load.value Current:100.00 b Max:100.00 b " - "Min:0.00 b ") + "Min:0.00 b ", + ) series = self._generate_series_list() - cacti = functions.cactiStyle({}, series, 'binary', 'b') + cacti = functions.cactiStyle({}, series, "binary", "b") self.assertEqual( cacti[0].name, "collectd.test-db1.load.value Current:100.00 b Max:100.00 b " - "Min:0.00 b ") + "Min:0.00 b ", + ) series = self._generate_series_list(config=[[None] * 100]) cacti = functions.cactiStyle({}, series) self.assertEqual( cacti[0].name, "collectd.test-db1.load.value Current:nan Max:nan " - "Min:nan ") + "Min:nan ", + ) cacti = functions.cactiStyle({}, []) self.assertEqual(cacti, []) def test_alias_by_metric(self): series = self._generate_series_list(config=[range(100)]) - series[0].name = 'scaleToSeconds(%s,10)' % series[0].name + series[0].name = "scaleToSeconds(%s,10)" % series[0].name alias = functions.aliasByMetric({}, series)[0] self.assertEqual(alias.name, "value") def test_legend_value(self): series = self._generate_series_list(config=[range(100)]) - legend = functions.legendValue({}, series, 'min', 'max', 'avg')[0] + legend = functions.legendValue({}, series, "min", "max", "avg")[0] self.assertEqual( legend.name, - "collectd.test-db1.load.value (min: 0) (max: 99) (avg: 49.5)") + "collectd.test-db1.load.value (min: 0) (max: 99) (avg: 49.5)", + ) series = self._generate_series_list(config=[range(100)]) - series[0].name = 'load.value' - legend = functions.legendValue({}, series, 'avg', 'si')[0] - self.assertEqual( - legend.name, - "load.value avg 49.50 ") + series[0].name = "load.value" + legend = functions.legendValue({}, series, "avg", "si")[0] + self.assertEqual(legend.name, "load.value avg 49.50 ") series = self._generate_series_list(config=[range(100)]) - legend = functions.legendValue({}, series, 'lol')[0] + legend = functions.legendValue({}, series, "lol")[0] self.assertEqual( - legend.name, "collectd.test-db1.load.value (lol: (?))") + legend.name, "collectd.test-db1.load.value (lol: (?))" + ) series = self._generate_series_list(config=[[None] * 100]) - legend = functions.legendValue({}, series, 'min')[0] + legend = functions.legendValue({}, series, "min")[0] self.assertEqual( - legend.name, "collectd.test-db1.load.value (min: None)") + legend.name, "collectd.test-db1.load.value (min: None)" + ) def test_substr(self): series = self._generate_series_list(config=[range(100)]) @@ -2143,8 +3096,10 @@ def test_substr(self): self.assertEqual(sub.name, "test-db1.load.value") series = functions.alias( - {}, self._generate_series_list(config=[range(100)]), - '(foo.bar, "baz")') + {}, + self._generate_series_list(config=[range(100)]), + '(foo.bar, "baz")', + ) sub = functions.substr({}, series, 1)[0] self.assertEqual(sub.name, "bar") @@ -2205,9 +3160,9 @@ def test_min_below_all_none(self): self.assertEqual(functions.minimumBelow({}, [[None]], 1), [[None]]) def test_highest_current(self): - series = self._generate_series_list(config=[range(100), - range(10, 110), - range(200, 300)]) + series = self._generate_series_list( + config=[range(100), range(10, 110), range(200, 300)] + ) highest = functions.highestCurrent({}, series)[0] self.assertEqual(highest.name, "collectd.test-db3.load.value") @@ -2215,9 +3170,9 @@ def test_highest_current(self): self.assertEqual(highest[0].name, "collectd.test-db2.load.value") def test_lowest_current(self): - series = self._generate_series_list(config=[range(100), - range(10, 110), - range(200, 300)]) + series = self._generate_series_list( + config=[range(100), range(10, 110), range(200, 300)] + ) lowest = functions.lowestCurrent({}, series)[0] self.assertEqual(lowest.name, "collectd.test-db1.load.value") @@ -2243,10 +3198,13 @@ def test_current_below_all_none(self): self.assertEqual(functions.currentBelow({}, [[None]], 1), [[None]]) def test_highest_average(self): - series = self._generate_series_list(config=[ - range(100), - range(50, 150), - list(range(150, 200)) + [None] * 50]) + series = self._generate_series_list( + config=[ + range(100), + range(50, 150), + list(range(150, 200)) + [None] * 50, + ] + ) highest = functions.highestAverage({}, series, 2) self.assertEqual(len(highest), 2) self.assertEqual(highest, [series[1], series[2]]) @@ -2255,10 +3213,13 @@ def test_highest_average(self): self.assertEqual(highest, [series[2]]) def test_lowest_average(self): - series = self._generate_series_list(config=[ - range(100), - range(50, 150), - list(range(150, 200)) + [None] * 50]) + series = self._generate_series_list( + config=[ + range(100), + range(50, 150), + list(range(150, 200)) + [None] * 50, + ] + ) lowest = functions.lowestAverage({}, series, 2) self.assertEqual(len(lowest), 2) self.assertEqual(lowest, [series[0], series[1]]) @@ -2290,7 +3251,8 @@ def test_average_below_all_none(self): def test_average_outside_percentile(self): series = self._generate_series_list( - config=[range(i, i+100) for i in range(50)]) + config=[range(i, i + 100) for i in range(50)] + ) outside = functions.averageOutsidePercentile({}, series, 95) self.assertEqual(outside, series[:3] + series[-2:]) @@ -2299,7 +3261,8 @@ def test_average_outside_percentile(self): def test_remove_between_percentile(self): series = self._generate_series_list( - config=[range(i, i+100) for i in range(50)]) + config=[range(i, i + 100) for i in range(50)] + ) not_between = functions.removeBetweenPercentile({}, series, 95) self.assertEqual(not_between, series[:3] + series[-2:]) @@ -2307,8 +3270,13 @@ def test_remove_between_percentile(self): self.assertEqual(not_between, series[:3] + series[-2:]) def test_sort_by_name(self): - series = list(reversed(self._generate_series_list( - config=[range(100) for i in range(10)]))) + series = list( + reversed( + self._generate_series_list( + config=[range(100) for i in range(10)] + ) + ) + ) sorted_s = functions.sortByName({}, series) self.assertEqual(sorted_s[0].name, series[-1].name) @@ -2317,43 +3285,55 @@ def test_sort_by_name(self): def test_sort_by_total(self): series = self._generate_series_list( - config=[range(i, i+100) for i in range(10)]) + config=[range(i, i + 100) for i in range(10)] + ) sorted_s = functions.sortByTotal({}, series) self.assertEqual(sorted_s[0].name, series[-1].name) def test_sort_by_maxima(self): - series = list(reversed(self._generate_series_list( - config=[range(i, i+100) for i in range(10)]))) + series = list( + reversed( + self._generate_series_list( + config=[range(i, i + 100) for i in range(10)] + ) + ) + ) sorted_s = functions.sortByMaxima({}, series) self.assertEqual(sorted_s[0].name, series[-1].name) def test_sort_by_minima(self): - series = list(reversed(self._generate_series_list( - config=[range(i, i+100) for i in range(10)]))) + series = list( + reversed( + self._generate_series_list( + config=[range(i, i + 100) for i in range(10)] + ) + ) + ) sorted_s = functions.sortByMinima({}, series) self.assertEqual(sorted_s[0].name, series[-1].name) def test_use_series_above(self): series = self._generate_series_list( - config=[list(range(90)) + [None] * 10]) - series[0].pathExpression = 'bar' + config=[list(range(90)) + [None] * 10] + ) + series[0].pathExpression = "bar" for s in series: self.write_series(s) - series[0].name = 'foo' + series[0].name = "foo" ctx = { - 'startTime': parseATTime('-100s'), - 'endTime': parseATTime('now'), + "startTime": parseATTime("-100s"), + "endTime": parseATTime("now"), } - above = functions.useSeriesAbove(ctx, series, 10, 'foo', 'bar')[0] + above = functions.useSeriesAbove(ctx, series, 10, "foo", "bar")[0] self.assertEqual(above[0], 2) - above = functions.useSeriesAbove(ctx, series, 100, 'foo', 'bar') + above = functions.useSeriesAbove(ctx, series, 100, "foo", "bar") self.assertEqual(len(above), 0) - above = functions.useSeriesAbove(ctx, series, 10, 'foo', 'baz') + above = functions.useSeriesAbove(ctx, series, 10, "foo", "baz") self.assertEqual(len(above), 0) def test_fallback_series(self): @@ -2362,27 +3342,59 @@ def test_fallback_series(self): self.assertEqual(functions.fallbackSeries({}, [], series), series) def test_most_deviant(self): - series = self._generate_series_list(config=[ - range(1, i * 100, i) for i in range(1, 10)] + [[None] * 100]) + series = self._generate_series_list( + config=[range(1, i * 100, i) for i in range(1, 10)] + + [[None] * 100] + ) deviant = functions.mostDeviant({}, series, 8) - self.assertEqual(deviant[0].name, 'collectd.test-db9.load.value') + self.assertEqual(deviant[0].name, "collectd.test-db9.load.value") def test_stdev(self): - series = self._generate_series_list(config=[ - [x**1.5 for x in range(100)], [None] * 100]) + series = self._generate_series_list( + config=[[x ** 1.5 for x in range(100)], [None] * 100] + ) dev = functions.stdev({}, series, 10)[0] self.assertEqual(dev[1], 0.5) def test_stdev_math_domain(self): # This data set is enough to trigger a math domain error in stdev. # The last two data points should be NaN. - inputData = [0.9, 0.1, 0.5, 0.7, 0.5, 0.4, 0.4, 0.3, 0.6, - 0.6333333333333333, 1.3, - 0.6333333333333333, 0.6185185185185186, - 0.3, 0.5, 0.6, 0.3, 1.0, 0.6, 0.9, 0.4, - 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] + inputData = [ + 0.9, + 0.1, + 0.5, + 0.7, + 0.5, + 0.4, + 0.4, + 0.3, + 0.6, + 0.6333333333333333, + 1.3, + 0.6333333333333333, + 0.6185185185185186, + 0.3, + 0.5, + 0.6, + 0.3, + 1.0, + 0.6, + 0.9, + 0.4, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + ] series = [ - TimeSeries('collectd.test-db0.load.value', 600, 760, 5, inputData) + TimeSeries("collectd.test-db0.load.value", 600, 760, 5, inputData) ] series[0].pathExpression = series[0].name @@ -2390,18 +3402,27 @@ def test_stdev_math_domain(self): self.assertEqual(result[-2:], [None, None]) def test_holt_winters_analysis_none(self): - seriesList = TimeSeries('collectd.test-db0.load.value', - 660, 700, 1, [None]) + seriesList = TimeSeries( + "collectd.test-db0.load.value", 660, 700, 1, [None] + ) expectedResults = { - 'predictions': TimeSeries( - 'holtWintersForecast(collectd.test-db0.load.value)', - 660, 700, 1, [None]), - 'deviations': TimeSeries( - 'holtWintersDeviation(collectd.test-db0.load.value)', - 660, 700, 1, [0]), - 'seasonals': [0], - 'slopes': [0], - 'intercepts': [None] + "predictions": TimeSeries( + "holtWintersForecast(collectd.test-db0.load.value)", + 660, + 700, + 1, + [None], + ), + "deviations": TimeSeries( + "holtWintersDeviation(collectd.test-db0.load.value)", + 660, + 700, + 1, + [0], + ), + "seasonals": [0], + "slopes": [0], + "intercepts": [None], } result = functions.holtWintersAnalysis(seriesList) @@ -2410,8 +3431,13 @@ def test_holt_winters_analysis_none(self): def test_holt_winters_forecast(self): def gen_series_list(start=0): seriesList = [ - TimeSeries('collectd.test-db0.load.value', - start+600, start+700, 1, range(start, start+100)), + TimeSeries( + "collectd.test-db0.load.value", + start + 600, + start + 700, + 1, + range(start, start + 100), + ) ] for series in seriesList: series.pathExpression = series.name @@ -2423,17 +3449,25 @@ def mock_evaluate(reqCtx, tokens, store=None): return gen_series_list() expectedResults = [ - TimeSeries('holtWintersForecast(collectd.test-db0.load.value)', - 605400, 700, 1, []) + TimeSeries( + "holtWintersForecast(collectd.test-db0.load.value)", + 605400, + 700, + 1, + [], + ) ] - with patch('graphite_api.functions.evaluateTokens', mock_evaluate): - result = functions.holtWintersForecast({ - 'args': ({}, {}), - 'startTime': datetime(1970, 2, 1, 0, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 2, 1, 0, 9, 0, 0, pytz.utc), - 'data': [], - }, series) + with patch("graphite_api.functions.evaluateTokens", mock_evaluate): + result = functions.holtWintersForecast( + { + "args": ({}, {}), + "startTime": datetime(1970, 2, 1, 0, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 2, 1, 0, 9, 0, 0, pytz.utc), + "data": [], + }, + series, + ) self.assertEqual(result, expectedResults) def test_holt_winters(self): @@ -2449,9 +3483,13 @@ def hw_range(x, y, jump): def gen_series_list(start=0, points=10): seriesList = [ - TimeSeries('collectd.test-db0.load.value', - start, start+(points*step), step, - hw_range(0, points*step, step)), + TimeSeries( + "collectd.test-db0.load.value", + start, + start + (points * step), + step, + hw_range(0, points * step, step), + ) ] for series in seriesList: series.pathExpression = series.name @@ -2460,176 +3498,235 @@ def gen_series_list(start=0, points=10): series = gen_series_list(start_time, points) def mock_evaluate(reqCtx, tokens, store=None): - return gen_series_list(start_time - week_seconds, - (week_seconds / step) + points) + return gen_series_list( + start_time - week_seconds, (week_seconds / step) + points + ) - expectedResults = [[ - TimeSeries( - 'holtWintersConfidenceLower(collectd.test-db0.load.value)', - start_time, start_time+(points*step), step, - [0.2841206166091448, 1.0581027098774411, 0.3338172102994683, - 0.5116859493263242, -0.18199175514936972, 0.2366173792019426, - -1.2941554508809152, -0.513426806531049, -0.7970905542723132, - 0.09868900726536012] - ), - TimeSeries( - 'holtWintersConfidenceUpper(collectd.test-db0.load.value)', - start_time, start_time+(points*step), step, - [8.424944558327624, 9.409422251880809, 10.607070189221787, - 10.288439865038768, 9.491556863132963, 9.474595784593738, - 8.572310478053845, 8.897670449095346, 8.941566968508148, - 9.409728797779282] - ), - ], [ - TimeSeries( - 'holtWintersConfidenceArea(collectd.test-db0.load.value)', - start_time, start_time+(points*step), step, - [0.2841206166091448, 1.0581027098774411, 0.3338172102994683, - 0.5116859493263242, -0.18199175514936972, 0.2366173792019426, - -1.2941554508809152, -0.513426806531049, -0.7970905542723132, - 0.09868900726536012] - ), - TimeSeries( - 'holtWintersConfidenceArea(collectd.test-db0.load.value)', - start_time, start_time+(points*step), step, - [8.424944558327624, 9.409422251880809, 10.607070189221787, - 10.288439865038768, 9.491556863132963, 9.474595784593738, - 8.572310478053845, 8.897670449095346, 8.941566968508148, - 9.409728797779282] - ), - ], [ - TimeSeries( - 'holtWintersAberration(collectd.test-db0.load.value)', - start_time, start_time+(points*step), step, - [-0.2841206166091448, -0.05810270987744115, - 0, 0, 0, 0, 0, 0, 0, 0] - ) - ]] - expectedResults[1][0].options = {'invisible': True, 'stacked': True} - expectedResults[1][1].options = {'stacked': True} - - with patch('graphite_api.functions.evaluateTokens', mock_evaluate): - result = functions.holtWintersConfidenceBands({ - 'args': ({}, {}), - 'startTime': datetime(1970, 2, 1, 0, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 2, 1, 0, 9, 0, 0, pytz.utc), - 'data': [] - }, series) + expectedResults = [ + [ + TimeSeries( + "holtWintersConfidenceLower(collectd.test-db0.load.value)", + start_time, + start_time + (points * step), + step, + [ + 0.2841206166091448, + 1.0581027098774411, + 0.3338172102994683, + 0.5116859493263242, + -0.18199175514936972, + 0.2366173792019426, + -1.2941554508809152, + -0.513426806531049, + -0.7970905542723132, + 0.09868900726536012, + ], + ), + TimeSeries( + "holtWintersConfidenceUpper(collectd.test-db0.load.value)", + start_time, + start_time + (points * step), + step, + [ + 8.424944558327624, + 9.409422251880809, + 10.607070189221787, + 10.288439865038768, + 9.491556863132963, + 9.474595784593738, + 8.572310478053845, + 8.897670449095346, + 8.941566968508148, + 9.409728797779282, + ], + ), + ], + [ + TimeSeries( + "holtWintersConfidenceArea(collectd.test-db0.load.value)", + start_time, + start_time + (points * step), + step, + [ + 0.2841206166091448, + 1.0581027098774411, + 0.3338172102994683, + 0.5116859493263242, + -0.18199175514936972, + 0.2366173792019426, + -1.2941554508809152, + -0.513426806531049, + -0.7970905542723132, + 0.09868900726536012, + ], + ), + TimeSeries( + "holtWintersConfidenceArea(collectd.test-db0.load.value)", + start_time, + start_time + (points * step), + step, + [ + 8.424944558327624, + 9.409422251880809, + 10.607070189221787, + 10.288439865038768, + 9.491556863132963, + 9.474595784593738, + 8.572310478053845, + 8.897670449095346, + 8.941566968508148, + 9.409728797779282, + ], + ), + ], + [ + TimeSeries( + "holtWintersAberration(collectd.test-db0.load.value)", + start_time, + start_time + (points * step), + step, + [ + -0.2841206166091448, + -0.05810270987744115, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + ], + ) + ], + ] + expectedResults[1][0].options = {"invisible": True, "stacked": True} + expectedResults[1][1].options = {"stacked": True} + + with patch("graphite_api.functions.evaluateTokens", mock_evaluate): + result = functions.holtWintersConfidenceBands( + { + "args": ({}, {}), + "startTime": datetime(1970, 2, 1, 0, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 2, 1, 0, 9, 0, 0, pytz.utc), + "data": [], + }, + series, + ) self.assertEqual(result, expectedResults[0]) - result = functions.holtWintersConfidenceArea({ - 'args': ({}, {}), - 'startTime': datetime(1970, 2, 1, 0, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 2, 1, 0, 9, 0, 0, pytz.utc), - 'data': [] - }, series) + result = functions.holtWintersConfidenceArea( + { + "args": ({}, {}), + "startTime": datetime(1970, 2, 1, 0, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 2, 1, 0, 9, 0, 0, pytz.utc), + "data": [], + }, + series, + ) self.assertEqual(result, expectedResults[1]) - result = functions.holtWintersAberration({ - 'args': ({}, {}), - 'startTime': datetime(1970, 2, 1, 0, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 2, 1, 0, 9, 0, 0, pytz.utc), - 'data': [] - }, series) + result = functions.holtWintersAberration( + { + "args": ({}, {}), + "startTime": datetime(1970, 2, 1, 0, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 2, 1, 0, 9, 0, 0, pytz.utc), + "data": [], + }, + series, + ) self.assertEqual(result, expectedResults[2]) def test_dashed(self): series = self._generate_series_list(config=[range(100)]) dashed = functions.dashed({}, series)[0] - self.assertEqual(dashed.options, {'dashed': 5}) + self.assertEqual(dashed.options, {"dashed": 5}) dashed = functions.dashed({}, series, 12)[0] - self.assertEqual(dashed.options, {'dashed': 12}) + self.assertEqual(dashed.options, {"dashed": 12}) def test_time_stack(self): timespan = 3600 * 24 * 8 # 8 days stop = int(time.time()) step = 100 - series = TimeSeries('foo.bar', - stop - timespan, - stop, - step, - [x**1.5 for x in range(0, timespan, step)]) + series = TimeSeries( + "foo.bar", + stop - timespan, + stop, + step, + [x ** 1.5 for x in range(0, timespan, step)], + ) series[10] = None - series.pathExpression = 'foo.bar' + series.pathExpression = "foo.bar" self.write_series(series, [(100, timespan)]) - ctx = {'startTime': parseATTime('-1d'), - 'endTime': parseATTime('now')} - stack = functions.timeStack(ctx, [series], '1d', 0, 7) + ctx = {"startTime": parseATTime("-1d"), "endTime": parseATTime("now")} + stack = functions.timeStack(ctx, [series], "1d", 0, 7) self.assertEqual(len(stack), 7) - stack = functions.timeStack(ctx, [series], '-1d', 0, 7) + stack = functions.timeStack(ctx, [series], "-1d", 0, 7) self.assertEqual(len(stack), 7) def test_time_shift(self): timespan = 3600 * 24 * 8 # 8 days stop = int(time.time()) step = 100 - series = TimeSeries('foo.bar', - stop - timespan, - stop, - step, - [x**1.5 for x in range(0, timespan, step)]) + series = TimeSeries( + "foo.bar", + stop - timespan, + stop, + step, + [x ** 1.5 for x in range(0, timespan, step)], + ) series[10] = None - series.pathExpression = 'foo.bar' + series.pathExpression = "foo.bar" self.write_series(series, [(100, timespan)]) - ctx = {'startTime': parseATTime('-1d'), - 'endTime': parseATTime('now')} - shift = functions.timeShift(ctx, [series], '1d') + ctx = {"startTime": parseATTime("-1d"), "endTime": parseATTime("now")} + shift = functions.timeShift(ctx, [series], "1d") self.assertEqual(len(shift), 1) - shift = functions.timeShift(ctx, [series], '-1d', False) + shift = functions.timeShift(ctx, [series], "-1d", False) self.assertEqual(len(shift), 1) - shift = functions.timeShift(ctx, [], '-1d') + shift = functions.timeShift(ctx, [], "-1d") self.assertEqual(len(shift), 0) def test_constant_line(self): - ctx = { - 'startTime': parseATTime('-1d'), - 'endTime': parseATTime('now'), - } + ctx = {"startTime": parseATTime("-1d"), "endTime": parseATTime("now")} line = functions.constantLine(ctx, 12)[0] self.assertEqual(list(line), [12, 12, 12]) self.assertEqual(line.step, 3600 * 24 // 2) def test_aggregate_line(self): - ctx = { - 'startTime': parseATTime('-1d'), - 'endTime': parseATTime('now'), - } + ctx = {"startTime": parseATTime("-1d"), "endTime": parseATTime("now")} series = self._generate_series_list(config=[range(100)]) line = functions.aggregateLine(ctx, series)[0] self.assertEqual(list(line), [49.5, 49.5, 49.5]) - line = functions.aggregateLine(ctx, series, 'avg')[0] + line = functions.aggregateLine(ctx, series, "avg")[0] self.assertEqual(list(line), [49.5, 49.5, 49.5]) - line = functions.aggregateLine(ctx, series, 'min')[0] + line = functions.aggregateLine(ctx, series, "min")[0] self.assertEqual(list(line), [0, 0, 0]) - line = functions.aggregateLine(ctx, series, 'max')[0] + line = functions.aggregateLine(ctx, series, "max")[0] self.assertEqual(list(line), [99, 99, 99]) with self.assertRaises(ValueError): - functions.aggregateLine(ctx, series, 'foo') + functions.aggregateLine(ctx, series, "foo") def test_threshold(self): - ctx = { - 'startTime': parseATTime('-1d'), - 'endTime': parseATTime('now'), - } - threshold = functions.threshold(ctx, 123, 'foobar')[0] + ctx = {"startTime": parseATTime("-1d"), "endTime": parseATTime("now")} + threshold = functions.threshold(ctx, 123, "foobar")[0] self.assertEqual(list(threshold), [123, 123, 123]) threshold = functions.threshold(ctx, 123)[0] self.assertEqual(list(threshold), [123, 123, 123]) - threshold = functions.threshold(ctx, 123, 'foo', 'red')[0] + threshold = functions.threshold(ctx, 123, "foo", "red")[0] self.assertEqual(list(threshold), [123, 123, 123]) - self.assertEqual(threshold.color, 'red') + self.assertEqual(threshold.color, "red") def test_non_null(self): one = [None, 0, 2, 3] * 25 @@ -2640,214 +3737,243 @@ def test_non_null(self): self.assertEqual(non_null[1][:5], [0, 1, 1, 0, 1]) def test_identity(self): - ctx = { - 'startTime': parseATTime('-1d'), - 'endTime': parseATTime('now'), - } - identity = functions.identity(ctx, 'foo')[0] + ctx = {"startTime": parseATTime("-1d"), "endTime": parseATTime("now")} + identity = functions.identity(ctx, "foo")[0] self.assertEqual(identity.end - identity.start, 3600 * 24) def test_count(self): - series = self._generate_series_list(config=[range(100), - range(100, 200)]) + series = self._generate_series_list( + config=[range(100), range(100, 200)] + ) count = functions.countSeries({}, series)[0] self.assertEqual(list(count), [2] * 100) def test_empty_count(self): - expectedResult = [ - TimeSeries('0', 0, 600, 300, [0, 0, 0]), - ] + expectedResult = [TimeSeries("0", 0, 600, 300, [0, 0, 0])] ctx = { - 'startTime': datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 1, 1, 0, 10, 0, 0, pytz.utc), + "startTime": datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 1, 1, 0, 10, 0, 0, pytz.utc), } result = functions.countSeries(ctx) self.assertEqual(result, expectedResult) def test_group_by_node(self): - series = self._generate_series_list(config=[range(100), - range(100, 200)]) - grouped = functions.groupByNode({}, series, 1, 'sumSeries') + series = self._generate_series_list( + config=[range(100), range(100, 200)] + ) + grouped = functions.groupByNode({}, series, 1, "sumSeries") first, second = grouped - self.assertEqual(first.name, 'test-db1') - self.assertEqual(second.name, 'test-db2') + self.assertEqual(first.name, "test-db1") + self.assertEqual(second.name, "test-db2") series[1].name = series[0].name - grouped = functions.groupByNode({}, series, 1, 'sumSeries') + grouped = functions.groupByNode({}, series, 1, "sumSeries") self.assertEqual(len(grouped), 1) - self.assertEqual(grouped[0].name, 'test-db1') + self.assertEqual(grouped[0].name, "test-db1") self.assertEqual(list(grouped[0])[:3], [100, 102, 104]) def test_group_by_nodes(self): seriesList = [ - TimeSeries('group1.server1.load5', 0, 2, 1, [10, 20]), - TimeSeries('group1.server1.load10', 0, 2, 1, [1, 2]), - TimeSeries('group1.server2.load5', 0, 2, 1, [30, 40]), - TimeSeries('group1.server2.load10', 0, 2, 1, [3, 4]), - TimeSeries('group2.server1.load5', 0, 2, 1, [40, 50]), - TimeSeries('group2.server1.load10', 0, 2, 1, [4, 5]), - TimeSeries('group2.server2.load5', 0, 2, 1, [60, 70]), - TimeSeries('group2.server2.load10', 0, 2, 1, [6, 7]), + TimeSeries("group1.server1.load5", 0, 2, 1, [10, 20]), + TimeSeries("group1.server1.load10", 0, 2, 1, [1, 2]), + TimeSeries("group1.server2.load5", 0, 2, 1, [30, 40]), + TimeSeries("group1.server2.load10", 0, 2, 1, [3, 4]), + TimeSeries("group2.server1.load5", 0, 2, 1, [40, 50]), + TimeSeries("group2.server1.load10", 0, 2, 1, [4, 5]), + TimeSeries("group2.server2.load5", 0, 2, 1, [60, 70]), + TimeSeries("group2.server2.load10", 0, 2, 1, [6, 7]), ] for series in seriesList: series.pathExpression = series.name - grouped = functions.groupByNodes({}, seriesList, - 'sumSeries', 0, 2) + grouped = functions.groupByNodes({}, seriesList, "sumSeries", 0, 2) - expectedNames = ['group1.load5', 'group1.load10', - 'group2.load5', 'group2.load10'] + expectedNames = [ + "group1.load5", + "group1.load10", + "group2.load5", + "group2.load10", + ] self.assertEqual([s.name for s in grouped], expectedNames) group1_load10 = grouped[1] self.assertEqual(list(group1_load10), [4, 6]) def test_exclude(self): - series = self._generate_series_list(config=[range(100), - range(100, 200)]) - excl = functions.exclude({}, series, 'db1') + series = self._generate_series_list( + config=[range(100), range(100, 200)] + ) + excl = functions.exclude({}, series, "db1") self.assertEqual(excl, [series[1]]) def test_grep(self): - series = self._generate_series_list(config=[range(100), - range(100, 200)]) - grep = functions.grep({}, series, 'db1') + series = self._generate_series_list( + config=[range(100), range(100, 200)] + ) + grep = functions.grep({}, series, "db1") self.assertEqual(grep, [series[0]]) def test_smart_summarize(self): ctx = { - 'startTime': parseATTime('-1min'), - 'endTime': parseATTime('now'), - 'tzinfo': pytz.timezone('UTC'), + "startTime": parseATTime("-1min"), + "endTime": parseATTime("now"), + "tzinfo": pytz.timezone("UTC"), } series = self._generate_series_list(config=[range(100)]) for s in series: self.write_series(s) - summ = functions.smartSummarize(ctx, series, '5s')[0] + summ = functions.smartSummarize(ctx, series, "5s")[0] self.assertEqual(summ[:3], [220, 245, 270]) - summ = functions.smartSummarize(ctx, series, '5s', 'avg')[0] + summ = functions.smartSummarize(ctx, series, "5s", "avg")[0] self.assertEqual(summ[:3], [44, 49, 54]) - summ = functions.smartSummarize(ctx, series, '5s', 'last')[0] + summ = functions.smartSummarize(ctx, series, "5s", "last")[0] self.assertEqual(summ[:3], [46, 51, 56]) - summ = functions.smartSummarize(ctx, series, '5s', 'max')[0] + summ = functions.smartSummarize(ctx, series, "5s", "max")[0] self.assertEqual(summ[:3], [46, 51, 56]) - summ = functions.smartSummarize(ctx, series, '5s', 'min')[0] + summ = functions.smartSummarize(ctx, series, "5s", "min")[0] self.assertEqual(summ[:3], [42, 47, 52]) # Higher time interval should not trigger timezone errors - functions.smartSummarize(ctx, series, '100s', 'min')[0] + functions.smartSummarize(ctx, series, "100s", "min")[0] def test_summarize(self): series = self._generate_series_list(config=[list(range(99)) + [None]]) # summarize is not consistent enough to allow testing exact output - functions.summarize({}, series, '5s')[0] - functions.summarize({}, series, '5s', 'avg', True)[0] - functions.summarize({}, series, '5s', 'last')[0] - functions.summarize({}, series, '5s', 'min')[0] - functions.summarize({}, series, '5s', 'max')[0] + functions.summarize({}, series, "5s")[0] + functions.summarize({}, series, "5s", "avg", True)[0] + functions.summarize({}, series, "5s", "last")[0] + functions.summarize({}, series, "5s", "min")[0] + functions.summarize({}, series, "5s", "max")[0] def test_hitcount(self): ctx = { - 'startTime': parseATTime('-1min'), - 'endTime': parseATTime('now'), - 'tzinfo': pytz.timezone('UTC'), + "startTime": parseATTime("-1min"), + "endTime": parseATTime("now"), + "tzinfo": pytz.timezone("UTC"), } series = self._generate_series_list(config=[list(range(99)) + [None]]) for s in series: self.write_series(s) - hit = functions.hitcount(ctx, series, '5s')[0] + hit = functions.hitcount(ctx, series, "5s")[0] self.assertEqual(hit[:3], [0, 15, 40]) - hit = functions.hitcount(ctx, series, '5s', True)[0] + hit = functions.hitcount(ctx, series, "5s", True)[0] self.assertEqual(hit[:3], [220, 245, 270]) try: - hit = functions.hitcount(ctx, series, '1min', True)[0] - hit = functions.hitcount(ctx, series, '1h', True)[0] - hit = functions.hitcount(ctx, series, '1d', True)[0] + hit = functions.hitcount(ctx, series, "1min", True)[0] + hit = functions.hitcount(ctx, series, "1h", True)[0] + hit = functions.hitcount(ctx, series, "1d", True)[0] except ValueError as e: self.fail("hitcount() raised ValueError: %s" % str(e)) def test_random_walk(self): ctx = { - 'startTime': parseATTime('-12h'), - 'endTime': parseATTime('now'), + "startTime": parseATTime("-12h"), + "endTime": parseATTime("now"), } - walk = functions.randomWalkFunction(ctx, 'foo')[0] + walk = functions.randomWalkFunction(ctx, "foo")[0] self.assertEqual(len(walk), 721) def test_null_zero_sum(self): s = TimeSeries("s", 0, 1, 1, [None]) - s.pathExpression = 's' + s.pathExpression = "s" [series] = functions.sumSeries({}, [s]) self.assertEqual(list(series), [None]) s = TimeSeries("s", 0, 1, 1, [None, 1]) - s.pathExpression = 's' + s.pathExpression = "s" t = TimeSeries("s", 0, 1, 1, [None, None]) - t.pathExpression = 't' + t.pathExpression = "t" [series] = functions.sumSeries({}, [s, t]) self.assertEqual(list(series), [None, 1]) def test_multiply_with_wildcards(self): s1 = [ - TimeSeries('web.host-1.avg-response.value', 0, 1, 1, [1, 10, 11]), - TimeSeries('web.host-2.avg-response.value', 0, 1, 1, [2, 20, 21]), - TimeSeries('web.host-3.avg-response.value', 0, 1, 1, [3, 30, 31]), - TimeSeries('web.host-4.avg-response.value', 0, 1, 1, [4, 40, 41]), + TimeSeries("web.host-1.avg-response.value", 0, 1, 1, [1, 10, 11]), + TimeSeries("web.host-2.avg-response.value", 0, 1, 1, [2, 20, 21]), + TimeSeries("web.host-3.avg-response.value", 0, 1, 1, [3, 30, 31]), + TimeSeries("web.host-4.avg-response.value", 0, 1, 1, [4, 40, 41]), ] s2 = [ - TimeSeries('web.host-4.total-request.value', 0, 1, 1, [4, 8, 12]), - TimeSeries('web.host-3.total-request.value', 0, 1, 1, [3, 7, 11]), - TimeSeries('web.host-1.total-request.value', 0, 1, 1, [1, 5, 9]), - TimeSeries('web.host-2.total-request.value', 0, 1, 1, [2, 6, 10]), + TimeSeries("web.host-4.total-request.value", 0, 1, 1, [4, 8, 12]), + TimeSeries("web.host-3.total-request.value", 0, 1, 1, [3, 7, 11]), + TimeSeries("web.host-1.total-request.value", 0, 1, 1, [1, 5, 9]), + TimeSeries("web.host-2.total-request.value", 0, 1, 1, [2, 6, 10]), ] expected = [ - TimeSeries('web.host-1', 0, 1, 1, [1, 50, 99]), - TimeSeries('web.host-2', 0, 1, 1, [4, 120, 210]), - TimeSeries('web.host-3', 0, 1, 1, [9, 210, 341]), - TimeSeries('web.host-4', 0, 1, 1, [16, 320, 492]), + TimeSeries("web.host-1", 0, 1, 1, [1, 50, 99]), + TimeSeries("web.host-2", 0, 1, 1, [4, 120, 210]), + TimeSeries("web.host-3", 0, 1, 1, [9, 210, 341]), + TimeSeries("web.host-4", 0, 1, 1, [16, 320, 492]), ] results = functions.multiplySeriesWithWildcards({}, s1 + s2, 2, 3) self.assertEqual(results, expected) def test_timeslice(self): series = [ - TimeSeries('test.value', 0, 600, 60, - [None, 1, 2, 3, None, 5, 6, None, 7, 8, 9]), + TimeSeries( + "test.value", + 0, + 600, + 60, + [None, 1, 2, 3, None, 5, 6, None, 7, 8, 9], + ) ] expected = [ - TimeSeries('timeSlice(test.value, 180, 480)', 0, 600, 60, - [None, None, None, 3, None, 5, 6, None, 7, None, None]), + TimeSeries( + "timeSlice(test.value, 180, 480)", + 0, + 600, + 60, + [None, None, None, 3, None, 5, 6, None, 7, None, None], + ) ] - results = functions.timeSlice({ - 'startTime': datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), - 'endTime': datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), - 'data': [], - }, series, '00:03 19700101', '00:08 19700101') + results = functions.timeSlice( + { + "startTime": datetime(1970, 1, 1, 0, 0, 0, 0, pytz.utc), + "endTime": datetime(1970, 1, 1, 0, 9, 0, 0, pytz.utc), + "data": [], + }, + series, + "00:03 19700101", + "00:08 19700101", + ) self.assertEqual(results, expected) def test_remove_emtpy(self): series = [ - TimeSeries('foo.bar', 0, 100, 10, - [None, None, None, 0, 0, 0, 1, 1, 1, None]), - TimeSeries('foo.baz', 0, 100, 10, [None] * 10), - TimeSeries('foo.blah', 0, 100, 10, - [None, None, None, 0, 0, 0, 0, 0, 0, None]), + TimeSeries( + "foo.bar", + 0, + 100, + 10, + [None, None, None, 0, 0, 0, 1, 1, 1, None], + ), + TimeSeries("foo.baz", 0, 100, 10, [None] * 10), + TimeSeries( + "foo.blah", + 0, + 100, + 10, + [None, None, None, 0, 0, 0, 0, 0, 0, None], + ), ] results = functions.removeEmptySeries({}, series) self.assertEqual(results, [series[0], series[2]]) def test_legend_value_with_system_preserves_sign(self): - series = [TimeSeries("foo", 0, 1, 1, [-10000, -20000, -30000, -40000])] + series = [ + TimeSeries("foo", 0, 1, 1, [-10000, -20000, -30000, -40000]) + ] [result] = functions.legendValue({}, series, "avg", "si") self.assertEqual(result.name, "foo avg -25.00K ") @@ -2858,25 +3984,39 @@ def test_linear_regression(self): # savedSeries would be eg: ('test.value', 180, 480, 60) savedStart = now - 1320 savedEnd = now - 1020 - savedSeries = TimeSeries('test.value', savedStart, savedEnd, 60, - [3, None, 5, 6, None, 8]) - savedSeries.pathExpression = 'test.value' + savedSeries = TimeSeries( + "test.value", savedStart, savedEnd, 60, [3, None, 5, 6, None, 8] + ) + savedSeries.pathExpression = "test.value" self.write_series(savedSeries, [(60, 60)]) # inputSeries would be eg: ('test.value', 1200, 1500, 60) inputStart = now - 300 - inputSeries = TimeSeries('test.value', inputStart, now, 60, - [123, None, None, 456, None, None, None]) - inputSeries.pathExpression = 'test.value' - - results = functions.linearRegression({ - 'startTime': parseATTime(str(inputStart)), - }, [inputSeries], str(savedStart), str(savedEnd)) + inputSeries = TimeSeries( + "test.value", + inputStart, + now, + 60, + [123, None, None, 456, None, None, None], + ) + inputSeries.pathExpression = "test.value" + + results = functions.linearRegression( + {"startTime": parseATTime(str(inputStart))}, + [inputSeries], + str(savedStart), + str(savedEnd), + ) expected = [ - TimeSeries('linearRegression(test.value, %s, %s)' % - (savedStart, savedEnd), inputStart, now, 60, - [20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0]) + TimeSeries( + "linearRegression(test.value, %s, %s)" + % (savedStart, savedEnd), + inputStart, + now, + 60, + [20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0], + ) ] # regression function calculated from datapoints on minutes 3 to 8 @@ -2887,30 +4027,40 @@ def test_apply_by_node(self): start = end - 3 seriesList = [ - TimeSeries('servers.s1.disk.bytes_used', - start, end, 1, [10, 20, 30]), - TimeSeries('servers.s1.disk.bytes_free', - start, end, 1, [90, 80, 70]), - TimeSeries('servers.s2.disk.bytes_used', - start, end, 1, [1, 2, 3]), - TimeSeries('servers.s2.disk.bytes_free', - start, end, 1, [99, 98, 97]) + TimeSeries( + "servers.s1.disk.bytes_used", start, end, 1, [10, 20, 30] + ), + TimeSeries( + "servers.s1.disk.bytes_free", start, end, 1, [90, 80, 70] + ), + TimeSeries( + "servers.s2.disk.bytes_used", start, end, 1, [1, 2, 3] + ), + TimeSeries( + "servers.s2.disk.bytes_free", start, end, 1, [99, 98, 97] + ), ] for series in seriesList: series.pathExpression = series.name self.write_series(series) expectedResults = [ - TimeSeries('servers.s1.disk.pct_used', - start, end, 1, [0.10, 0.20, 0.30]), - TimeSeries('servers.s2.disk.pct_used', start, end, 1, - [0.01, 0.02, 0.03]) + TimeSeries( + "servers.s1.disk.pct_used", start, end, 1, [0.10, 0.20, 0.30] + ), + TimeSeries( + "servers.s2.disk.pct_used", start, end, 1, [0.01, 0.02, 0.03] + ), ] - result = functions.applyByNode({ - 'startTime': parseATTime(str(start - 1)), - 'endTime': parseATTime(str(end - 1)), - }, seriesList, 1, - 'divideSeries(%.disk.bytes_used, sumSeries(%.disk.bytes_*))', - '%.disk.pct_used') + result = functions.applyByNode( + { + "startTime": parseATTime(str(start - 1)), + "endTime": parseATTime(str(end - 1)), + }, + seriesList, + 1, + "divideSeries(%.disk.bytes_used, sumSeries(%.disk.bytes_*))", + "%.disk.pct_used", + ) self.assertEqual(result, expectedResults) diff --git a/tests/test_http.py b/tests/test_http.py index 3754a8b..684713f 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -3,31 +3,33 @@ class HttpTestCase(TestCase): def test_cors(self): - response = self.app.options('/render') - self.assertFalse( - 'Access-Control-Allow-Origin' in response.headers) + response = self.app.options("/render") + self.assertFalse("Access-Control-Allow-Origin" in response.headers) - response = self.app.options('/render', headers=( - ('Origin', 'https://example.com'), - )) - self.assertEqual(response.headers['Access-Control-Allow-Origin'], - 'https://example.com') + response = self.app.options( + "/render", headers=(("Origin", "https://example.com"),) + ) + self.assertEqual( + response.headers["Access-Control-Allow-Origin"], + "https://example.com", + ) - response = self.app.options('/render', headers=( - ('Origin', 'http://foo.example.com:8888'), - )) - self.assertEqual(response.headers['Access-Control-Allow-Origin'], - 'http://foo.example.com:8888') + response = self.app.options( + "/render", headers=(("Origin", "http://foo.example.com:8888"),) + ) + self.assertEqual( + response.headers["Access-Control-Allow-Origin"], + "http://foo.example.com:8888", + ) - response = self.app.options('/', headers=( - ('Origin', 'http://foo.example.com'), - )) - self.assertFalse( - 'Access-Control-Allow-Origin' in response.headers) + response = self.app.options( + "/", headers=(("Origin", "http://foo.example.com"),) + ) + self.assertFalse("Access-Control-Allow-Origin" in response.headers) def test_trailing_slash(self): - response = self.app.get('/render?target=foo') + response = self.app.get("/render?target=foo") self.assertEqual(response.status_code, 200) - response = self.app.get('/render/?target=foo') + response = self.app.get("/render/?target=foo") self.assertEqual(response.status_code, 200) diff --git a/tests/test_intervals.py b/tests/test_intervals.py index b1a1827..4a1fb82 100644 --- a/tests/test_intervals.py +++ b/tests/test_intervals.py @@ -11,7 +11,7 @@ def test_interval(self): i = Interval(0, 1) j = Interval(1, 2) k = Interval(0, 1) - l = Interval(0, 0) + m = Interval(0, 0) self.assertNotEqual(i, j) self.assertEqual(i, k) self.assertEqual(hash(i), hash(k)) @@ -22,9 +22,9 @@ def test_interval(self): self.assertTrue(j > i) self.assertTrue(i) - self.assertFalse(l) + self.assertFalse(m) - self.assertEqual(repr(i), '') + self.assertEqual(repr(i), "") self.assertIsNone(i.intersect(j)) self.assertEqual(i.intersect(k), k) @@ -32,17 +32,16 @@ def test_interval(self): self.assertEqual(i.union(j), Interval(0, 2)) with self.assertRaises(TypeError): - j.union(l) + j.union(m) - self.assertEqual(union_overlapping([i, j, k, l]), - [Interval(0, 2)]) + self.assertEqual(union_overlapping([i, j, k, m]), [Interval(0, 2)]) def test_interval_set(self): i = Interval(0, 1) j = Interval(1, 2) s = IntervalSet([i, j]) - self.assertEqual(repr(s), '[]') + self.assertEqual(repr(s), "[]") s = IntervalSet([i, j], disjoint=True) it = iter(s) @@ -52,10 +51,11 @@ def test_interval_set(self): self.assertTrue(s) self.assertFalse(IntervalSet([])) - self.assertEqual(s - IntervalSet([i]), - IntervalSet([j])) + self.assertEqual(s - IntervalSet([i]), IntervalSet([j])) self.assertFalse(IntervalSet([]).intersect(s)) - self.assertEqual(s.union(IntervalSet([Interval(3, 4)])), - IntervalSet([Interval(3, 4), i, j])) + self.assertEqual( + s.union(IntervalSet([Interval(3, 4)])), + IntervalSet([Interval(3, 4), i, j]), + ) diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 21e651d..8409d07 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -11,9 +11,9 @@ class MetricsTests(TestCase): def _create_dbs(self, ts=None): ts = ts or int(time.time()) for db in ( - ('test', 'foo.wsp'), - ('test', 'wat', 'welp.wsp'), - ('test', 'bar', 'baz.wsp'), + ("test", "foo.wsp"), + ("test", "wat", "welp.wsp"), + ("test", "bar", "baz.wsp"), ): db_path = os.path.join(WHISPER_DIR, *db) os.makedirs(os.path.dirname(db_path)) @@ -22,254 +22,320 @@ def _create_dbs(self, ts=None): whisper.update(db_path, 2, ts) def test_find(self): - url = '/metrics/find' + url = "/metrics/find" response = self.app.get(url) self.assertEqual(response.status_code, 400) - response = self.app.get(url, query_string={'query': 'test'}) + response = self.app.get(url, query_string={"query": "test"}) self.assertJSON(response, []) - response = self.app.get(url, query_string={'query': 'test', - 'format': 'completer'}) - self.assertJSON(response, {'metrics': []}) + response = self.app.get( + url, query_string={"query": "test", "format": "completer"} + ) + self.assertJSON(response, {"metrics": []}) - response = self.app.get(url, query_string={'query': 'test', - 'format': 'nodelist'}) - self.assertJSON(response, {'nodes': []}) + response = self.app.get( + url, query_string={"query": "test", "format": "nodelist"} + ) + self.assertJSON(response, {"nodes": []}) ts = int(time.time()) self._create_dbs(ts) - for _url in ['/metrics/find', '/metrics']: - response = self.app.get(_url, query_string={'query': 'test.*', - 'format': 'treejson'}) - self.assertJSON(response, [{ - 'allowChildren': 1, - 'expandable': 1, - 'id': 'test.bar', - 'leaf': 0, - 'text': 'bar', - }, { - 'allowChildren': 1, - 'expandable': 1, - 'id': 'test.wat', - 'leaf': 0, - 'text': 'wat', - }, { - 'allowChildren': 0, - 'expandable': 0, - 'id': 'test.foo', - 'leaf': 1, - 'text': 'foo', - }]) - - response = self.app.get(url, query_string={'query': 'test.*', - 'format': 'treejson', - 'wildcards': 1}) - self.assertJSON(response, [{ - 'text': '*', - 'expandable': 1, - 'leaf': 0, - 'id': 'test.*', - 'allowChildren': 1, - }, { - 'allowChildren': 1, - 'expandable': 1, - 'id': 'test.bar', - 'leaf': 0, - 'text': 'bar', - }, { - 'allowChildren': 1, - 'expandable': 1, - 'id': 'test.wat', - 'leaf': 0, - 'text': 'wat', - }, { - 'allowChildren': 0, - 'expandable': 0, - 'id': 'test.foo', - 'leaf': 1, - 'text': 'foo', - }]) - - response = self.app.get(url, query_string={'query': 'test.*', - 'format': 'completer'}) - self.assertJSON(response, {'metrics': [{ - 'is_leaf': 0, - 'name': 'bar', - 'path': 'test.bar.', - }, { - 'is_leaf': 1, - 'name': 'foo', - 'path': 'test.foo', - }, { - 'is_leaf': 0, - 'name': 'wat', - 'path': 'test.wat.', - }]}) - - response = self.app.get(url, query_string={'query': 'test.*', - 'wildcards': 1, - 'format': 'completer'}) - self.assertJSON(response, {'metrics': [{ - 'is_leaf': 0, - 'name': 'bar', - 'path': 'test.bar.', - }, { - 'is_leaf': 1, - 'name': 'foo', - 'path': 'test.foo', - }, { - 'is_leaf': 0, - 'name': 'wat', - 'path': 'test.wat.', - }, { - 'name': '*', - }]}) - - response = self.app.get(url, query_string={'query': 'test.*', - 'format': 'json'}) - data = json.loads(response.data.decode('utf-8')) + for _url in ["/metrics/find", "/metrics"]: + response = self.app.get( + _url, query_string={"query": "test.*", "format": "treejson"} + ) + self.assertJSON( + response, + [ + { + "allowChildren": 1, + "expandable": 1, + "id": "test.bar", + "leaf": 0, + "text": "bar", + }, + { + "allowChildren": 1, + "expandable": 1, + "id": "test.wat", + "leaf": 0, + "text": "wat", + }, + { + "allowChildren": 0, + "expandable": 0, + "id": "test.foo", + "leaf": 1, + "text": "foo", + }, + ], + ) + + response = self.app.get( + url, + query_string={ + "query": "test.*", + "format": "treejson", + "wildcards": 1, + }, + ) + self.assertJSON( + response, + [ + { + "text": "*", + "expandable": 1, + "leaf": 0, + "id": "test.*", + "allowChildren": 1, + }, + { + "allowChildren": 1, + "expandable": 1, + "id": "test.bar", + "leaf": 0, + "text": "bar", + }, + { + "allowChildren": 1, + "expandable": 1, + "id": "test.wat", + "leaf": 0, + "text": "wat", + }, + { + "allowChildren": 0, + "expandable": 0, + "id": "test.foo", + "leaf": 1, + "text": "foo", + }, + ], + ) + + response = self.app.get( + url, query_string={"query": "test.*", "format": "completer"} + ) + self.assertJSON( + response, + { + "metrics": [ + {"is_leaf": 0, "name": "bar", "path": "test.bar."}, + {"is_leaf": 1, "name": "foo", "path": "test.foo"}, + {"is_leaf": 0, "name": "wat", "path": "test.wat."}, + ] + }, + ) + + response = self.app.get( + url, + query_string={ + "query": "test.*", + "wildcards": 1, + "format": "completer", + }, + ) + self.assertJSON( + response, + { + "metrics": [ + {"is_leaf": 0, "name": "bar", "path": "test.bar."}, + {"is_leaf": 1, "name": "foo", "path": "test.foo"}, + {"is_leaf": 0, "name": "wat", "path": "test.wat."}, + {"name": "*"}, + ] + }, + ) + + response = self.app.get( + url, query_string={"query": "test.*", "format": "json"} + ) + data = json.loads(response.data.decode("utf-8")) self.assertEqual(len(data), 3) - self.assertEqual(data[0]['is_leaf'], False) - self.assertEqual(len(data[0]['intervals']), 0) - self.assertEqual(data[0]['path'], 'test.bar') + self.assertEqual(data[0]["is_leaf"], False) + self.assertEqual(len(data[0]["intervals"]), 0) + self.assertEqual(data[0]["path"], "test.bar") - self.assertEqual(data[1]['is_leaf'], True) - self.assertEqual(len(data[1]['intervals']), 1) + self.assertEqual(data[1]["is_leaf"], True) + self.assertEqual(len(data[1]["intervals"]), 1) # Adjustment by 1 is a race condition. - self.assertTrue(int(data[1]['intervals'][0]['start']) in [ts - 60, - ts - 59]) - self.assertTrue(int(data[1]['intervals'][0]['end']) in [ts, ts + 1]) - self.assertEqual(data[1]['path'], 'test.foo') - - self.assertEqual(data[2]['is_leaf'], False) - self.assertEqual(len(data[2]['intervals']), 0) - self.assertEqual(data[2]['path'], 'test.wat') - - response = self.app.get(url, query_string={'query': '*', - 'jsonp': 'foo', - 'format': 'json'}) - data = response.data.decode('utf-8') - self.assertEqual(json.loads(data.split("(")[1].strip(")")), - [{'is_leaf': False, 'intervals': [], 'path': 'test'}]) - - response = self.app.get(url, query_string={'query': '*', - 'format': 'nodelist'}) - self.assertJSON(response, {'nodes': ['test']}) - - response = self.app.get(url, query_string={'query': '*.*', - 'format': 'nodelist'}) - self.assertJSON(response, {'nodes': ['bar', 'foo', 'wat']}) - - response = self.app.get(url, query_string={'query': '*.*.*', - 'format': 'nodelist'}) - self.assertJSON(response, {'nodes': ['baz', 'welp']}) - - response = self.app.get(url, query_string={'query': '*.*.*', - 'format': 'nodelist', - 'position': '0'}) - self.assertJSON(response, {'nodes': ['test']}) + self.assertTrue( + int(data[1]["intervals"][0]["start"]) in [ts - 60, ts - 59] + ) + self.assertTrue(int(data[1]["intervals"][0]["end"]) in [ts, ts + 1]) + self.assertEqual(data[1]["path"], "test.foo") + + self.assertEqual(data[2]["is_leaf"], False) + self.assertEqual(len(data[2]["intervals"]), 0) + self.assertEqual(data[2]["path"], "test.wat") + + response = self.app.get( + url, query_string={"query": "*", "jsonp": "foo", "format": "json"} + ) + data = response.data.decode("utf-8") + self.assertEqual( + json.loads(data.split("(")[1].strip(")")), + [{"is_leaf": False, "intervals": [], "path": "test"}], + ) + + response = self.app.get( + url, query_string={"query": "*", "format": "nodelist"} + ) + self.assertJSON(response, {"nodes": ["test"]}) + + response = self.app.get( + url, query_string={"query": "*.*", "format": "nodelist"} + ) + self.assertJSON(response, {"nodes": ["bar", "foo", "wat"]}) + + response = self.app.get( + url, query_string={"query": "*.*.*", "format": "nodelist"} + ) + self.assertJSON(response, {"nodes": ["baz", "welp"]}) + + response = self.app.get( + url, + query_string={ + "query": "*.*.*", + "format": "nodelist", + "position": "0", + }, + ) + self.assertJSON(response, {"nodes": ["test"]}) def test_find_validation(self): - url = '/metrics/find' - response = self.app.get(url, query_string={'query': 'foo', - 'wildcards': 'aaa'}) - self.assertJSON(response, {'errors': {'wildcards': 'must be 0 or 1.'}}, - status_code=400) - - response = self.app.get(url, query_string={'query': 'foo', - 'from': 'aaa', - 'until': 'bbb'}) - self.assertJSON(response, {'errors': { - 'from': 'must be an epoch timestamp.', - 'until': 'must be an epoch timestamp.', - }}, status_code=400) - - response = self.app.get(url, query_string={'query': 'foo', - 'format': 'other'}) - self.assertJSON(response, {'errors': { - 'format': 'unrecognized format: "other".', - }}, status_code=400) + url = "/metrics/find" + response = self.app.get( + url, query_string={"query": "foo", "wildcards": "aaa"} + ) + self.assertJSON( + response, + {"errors": {"wildcards": "must be 0 or 1."}}, + status_code=400, + ) + + response = self.app.get( + url, query_string={"query": "foo", "from": "aaa", "until": "bbb"} + ) + self.assertJSON( + response, + { + "errors": { + "from": "must be an epoch timestamp.", + "until": "must be an epoch timestamp.", + } + }, + status_code=400, + ) + + response = self.app.get( + url, query_string={"query": "foo", "format": "other"} + ) + self.assertJSON( + response, + {"errors": {"format": 'unrecognized format: "other".'}}, + status_code=400, + ) def test_expand(self): - url = '/metrics/expand' + url = "/metrics/expand" response = self.app.get(url) - self.assertJSON(response, {'errors': - {'query': 'this parameter is required.'}}, - status_code=400) + self.assertJSON( + response, + {"errors": {"query": "this parameter is required."}}, + status_code=400, + ) - response = self.app.get(url, query_string={'query': 'test'}) - self.assertJSON(response, {'results': []}) + response = self.app.get(url, query_string={"query": "test"}) + self.assertJSON(response, {"results": []}) self._create_dbs() - response = self.app.get(url, query_string={'query': 'test'}) - self.assertJSON(response, {'results': ['test']}) - - response = self.app.get(url, query_string={'query': 'test.*'}) - self.assertJSON(response, {'results': ['test.bar', 'test.foo', - 'test.wat']}) - - response = self.app.get(url, query_string={'query': 'test.*', - 'leavesOnly': 1}) - self.assertJSON(response, {'results': ['test.foo']}) - - response = self.app.get(url, query_string={'query': 'test.*', - 'groupByExpr': 1}) - self.assertJSON(response, {'results': {'test.*': ['test.bar', - 'test.foo', - 'test.wat']}}) + response = self.app.get(url, query_string={"query": "test"}) + self.assertJSON(response, {"results": ["test"]}) + + response = self.app.get(url, query_string={"query": "test.*"}) + self.assertJSON( + response, {"results": ["test.bar", "test.foo", "test.wat"]} + ) + + response = self.app.get( + url, query_string={"query": "test.*", "leavesOnly": 1} + ) + self.assertJSON(response, {"results": ["test.foo"]}) + + response = self.app.get( + url, query_string={"query": "test.*", "groupByExpr": 1} + ) + self.assertJSON( + response, + {"results": {"test.*": ["test.bar", "test.foo", "test.wat"]}}, + ) def test_expand_validation(self): - url = '/metrics/expand' - response = self.app.get(url, query_string={'query': 'foo', - 'leavesOnly': 'bbb', - 'groupByExpr': 'aaa'}) - self.assertJSON(response, {'errors': { - 'groupByExpr': 'must be 0 or 1.', - 'leavesOnly': 'must be 0 or 1.', - }}, status_code=400) + url = "/metrics/expand" + response = self.app.get( + url, + query_string={ + "query": "foo", + "leavesOnly": "bbb", + "groupByExpr": "aaa", + }, + ) + self.assertJSON( + response, + { + "errors": { + "groupByExpr": "must be 0 or 1.", + "leavesOnly": "must be 0 or 1.", + } + }, + status_code=400, + ) def test_noop(self): - url = '/dashboard/find' + url = "/dashboard/find" response = self.app.get(url) - self.assertJSON(response, {'dashboards': []}) + self.assertJSON(response, {"dashboards": []}) - url = '/dashboard/load/foo' + url = "/dashboard/load/foo" response = self.app.get(url) - self.assertJSON(response, {'error': "Dashboard 'foo' does not exist."}, - status_code=404) + self.assertJSON( + response, + {"error": "Dashboard 'foo' does not exist."}, + status_code=404, + ) - url = '/events/get_data' + url = "/events/get_data" response = self.app.get(url) self.assertJSON(response, []) def test_metrics_index(self): - url = '/metrics/index.json' + url = "/metrics/index.json" response = self.app.get(url) self.assertJSON(response, []) - self.assertEqual(response.headers['Content-Type'], 'application/json') + self.assertEqual(response.headers["Content-Type"], "application/json") - response = self.app.get(url, query_string={'jsonp': 'foo'}) - self.assertEqual(response.data, b'foo([])') - self.assertEqual(response.headers['Content-Type'], 'text/javascript') + response = self.app.get(url, query_string={"jsonp": "foo"}) + self.assertEqual(response.data, b"foo([])") + self.assertEqual(response.headers["Content-Type"], "text/javascript") - parent = os.path.join(WHISPER_DIR, 'collectd') + parent = os.path.join(WHISPER_DIR, "collectd") os.makedirs(parent) - for metric in ['load', 'memory', 'cpu']: - db = os.path.join(parent, '{0}.wsp'.format(metric)) + for metric in ["load", "memory", "cpu"]: + db = os.path.join(parent, "{0}.wsp".format(metric)) whisper.create(db, [(1, 60)]) response = self.app.get(url) - self.assertJSON(response, [ - u'collectd.cpu', - u'collectd.load', - u'collectd.memory', - ]) - response = self.app.get(url, query_string={'jsonp': 'bar'}) + self.assertJSON( + response, [u"collectd.cpu", u"collectd.load", u"collectd.memory"] + ) + response = self.app.get(url, query_string={"jsonp": "bar"}) self.assertEqual( response.data, - b'bar(["collectd.cpu", "collectd.load", "collectd.memory"])') + b'bar(["collectd.cpu", "collectd.load", "collectd.memory"])', + ) diff --git a/tests/test_paths.py b/tests/test_paths.py index c928a6f..30cc3b1 100644 --- a/tests/test_paths.py +++ b/tests/test_paths.py @@ -8,6 +8,7 @@ class PathsTest(TestCase): TestCase for pathsFromTarget function """ + def validate_paths(self, expected, test): """ Assert the lengths of the expected list and the test list are the same @@ -27,7 +28,7 @@ def test_simple(self): Tests a target containing a single path expression. """ - target = 'test.simple.metric' + target = "test.simple.metric" expected = [target] self.validate_paths(expected, pathsFromTarget({}, target)) @@ -37,9 +38,9 @@ def test_func_args(self): arguments. """ - path_1 = 'test.1.metric' - path_2 = 'test.2.metric' - target = 'sumSeries(%s,%s)' % (path_1, path_2) + path_1 = "test.1.metric" + path_2 = "test.2.metric" + target = "sumSeries(%s,%s)" % (path_1, path_2) expected = [path_1, path_2] self.validate_paths(expected, pathsFromTarget({}, target)) @@ -49,9 +50,9 @@ def test_func_kwargs(self): a kwarg. """ - path_a = 'test.a.metric' - path_b = 'test.b.metric' - target = 'someFunc(%s,b=%s)' % (path_a, path_b) + path_a = "test.a.metric" + path_b = "test.b.metric" + target = "someFunc(%s,b=%s)" % (path_a, path_b) expected = [path_a, path_b] self.validate_paths(expected, pathsFromTarget({}, target)) @@ -62,11 +63,11 @@ def test_func_nested(self): """ paths = ( - 'test.a.metric', - 'test.b.metric', - 'test.c.metric', - 'test.d.metric', + "test.a.metric", + "test.b.metric", + "test.c.metric", + "test.d.metric", ) - target = 'outerFunc(innerFunc(%s, %s), s=innerFunc(%s, %s))' % paths + target = "outerFunc(innerFunc(%s, %s), s=innerFunc(%s, %s))" % paths expected = list(paths) self.validate_paths(expected, pathsFromTarget({}, target)) diff --git a/tests/test_render.py b/tests/test_render.py index 6ae5690..06649fa 100644 --- a/tests/test_render.py +++ b/tests/test_render.py @@ -14,8 +14,8 @@ class RenderTest(TestCase): - db = os.path.join(WHISPER_DIR, 'test.wsp') - url = '/render' + db = os.path.join(WHISPER_DIR, "test.wsp") + url = "/render" def create_db(self): whisper.create(self.db, [(1, 60)]) @@ -26,330 +26,489 @@ def create_db(self): whisper.update(self.db, 1.5, self.ts) def test_render_view(self): - response = self.app.get(self.url, query_string={'target': 'test', - 'format': 'json', - 'noCache': 'true'}) - self.assertEqual(json.loads(response.data.decode('utf-8')), []) - - response = self.app.get(self.url, query_string={'target': 'test', - 'format': 'raw', - 'noCache': 'true'}) - self.assertEqual(response.data.decode('utf-8'), "") - self.assertEqual(response.headers['Content-Type'], 'text/plain') - - response = self.app.get(self.url, query_string={'target': 'test', - 'format': 'pdf'}) - self.assertEqual(response.headers['Content-Type'], 'application/x-pdf') - - response = self.app.get(self.url, query_string={'target': 'test'}) - self.assertEqual(response.headers['Content-Type'], 'image/png') - - response = self.app.get(self.url, query_string={'target': 'test', - 'format': 'dygraph', - 'noCache': 'true'}) - self.assertEqual(json.loads(response.data.decode('utf-8')), {}) - - response = self.app.get(self.url, query_string={'target': 'test', - 'format': 'rickshaw', - 'noCache': 'true'}) - self.assertEqual(json.loads(response.data.decode('utf-8')), []) + response = self.app.get( + self.url, + query_string={ + "target": "test", + "format": "json", + "noCache": "true", + }, + ) + self.assertEqual(json.loads(response.data.decode("utf-8")), []) + + response = self.app.get( + self.url, + query_string={ + "target": "test", + "format": "raw", + "noCache": "true", + }, + ) + self.assertEqual(response.data.decode("utf-8"), "") + self.assertEqual(response.headers["Content-Type"], "text/plain") + + response = self.app.get( + self.url, query_string={"target": "test", "format": "pdf"} + ) + self.assertEqual( + response.headers["Content-Type"], "application/x-pdf" + ) + + response = self.app.get(self.url, query_string={"target": "test"}) + self.assertEqual(response.headers["Content-Type"], "image/png") + + response = self.app.get( + self.url, + query_string={ + "target": "test", + "format": "dygraph", + "noCache": "true", + }, + ) + self.assertEqual(json.loads(response.data.decode("utf-8")), {}) + + response = self.app.get( + self.url, + query_string={ + "target": "test", + "format": "rickshaw", + "noCache": "true", + }, + ) + self.assertEqual(json.loads(response.data.decode("utf-8")), []) self.create_db() - response = self.app.get(self.url, query_string={'target': 'test', - 'format': 'json'}) - data = json.loads(response.data.decode('utf-8')) - end = data[0]['datapoints'][-4:] + response = self.app.get( + self.url, query_string={"target": "test", "format": "json"} + ) + data = json.loads(response.data.decode("utf-8")) + end = data[0]["datapoints"][-4:] try: self.assertEqual( - end, [[None, self.ts - 3], [1.0, self.ts - 2], - [0.5, self.ts - 1], [1.5, self.ts]]) + end, + [ + [None, self.ts - 3], + [1.0, self.ts - 2], + [0.5, self.ts - 1], + [1.5, self.ts], + ], + ) except AssertionError: self.assertEqual( - end, [[1.0, self.ts - 2], [0.5, self.ts - 1], - [1.5, self.ts], [None, self.ts + 1]]) + end, + [ + [1.0, self.ts - 2], + [0.5, self.ts - 1], + [1.5, self.ts], + [None, self.ts + 1], + ], + ) - response = self.app.get(self.url, query_string={'target': 'test', - 'maxDataPoints': 2, - 'format': 'json'}) - data = json.loads(response.data.decode('utf-8')) + response = self.app.get( + self.url, + query_string={ + "target": "test", + "maxDataPoints": 2, + "format": "json", + }, + ) + data = json.loads(response.data.decode("utf-8")) # 1 is a time race cond - self.assertTrue(len(data[0]['datapoints']) in [1, 2]) + self.assertTrue(len(data[0]["datapoints"]) in [1, 2]) - response = self.app.get(self.url, query_string={'target': 'test', - 'maxDataPoints': 200, - 'format': 'json'}) - data = json.loads(response.data.decode('utf-8')) + response = self.app.get( + self.url, + query_string={ + "target": "test", + "maxDataPoints": 200, + "format": "json", + }, + ) + data = json.loads(response.data.decode("utf-8")) # 59 is a time race cond - self.assertTrue(len(data[0]['datapoints']) in [59, 60]) - - response = self.app.get(self.url, query_string={'target': 'test', - 'noNullPoints': 1, - 'format': 'json'}) - data = json.loads(response.data.decode('utf-8')) - self.assertEqual(data[0]['datapoints'], - [[1.0, self.ts - 2], - [0.5, self.ts - 1], - [1.5, self.ts]]) - - response = self.app.get(self.url, query_string={'target': 'test', - 'format': 'raw'}) + self.assertTrue(len(data[0]["datapoints"]) in [59, 60]) + + response = self.app.get( + self.url, + query_string={ + "target": "test", + "noNullPoints": 1, + "format": "json", + }, + ) + data = json.loads(response.data.decode("utf-8")) + self.assertEqual( + data[0]["datapoints"], + [[1.0, self.ts - 2], [0.5, self.ts - 1], [1.5, self.ts]], + ) + + response = self.app.get( + self.url, query_string={"target": "test", "format": "raw"} + ) try: self.assertEqual( - response.data.decode('utf-8'), - 'test,%d,%d,1|%s' % (self.ts - 59, self.ts + 1, - 'None,' * 57 + '1.0,0.5,1.5\n')) + response.data.decode("utf-8"), + "test,%d,%d,1|%s" + % (self.ts - 59, self.ts + 1, "None," * 57 + "1.0,0.5,1.5\n"), + ) except AssertionError: self.assertEqual( - response.data.decode('utf-8'), - 'test,%d,%d,1|%s' % (self.ts - 58, self.ts + 2, - 'None,' * 56 + '1.0,0.5,1.5,None\n')) - - response = self.app.get(self.url, query_string={'target': 'test', - 'format': 'dygraph'}) - data = json.loads(response.data.decode('utf-8')) - end = data['data'][-4:] + response.data.decode("utf-8"), + "test,%d,%d,1|%s" + % ( + self.ts - 58, + self.ts + 2, + "None," * 56 + "1.0,0.5,1.5,None\n", + ), + ) + + response = self.app.get( + self.url, query_string={"target": "test", "format": "dygraph"} + ) + data = json.loads(response.data.decode("utf-8")) + end = data["data"][-4:] try: self.assertEqual( - end, [[(self.ts - 3) * 1000, None], - [(self.ts - 2) * 1000, 1.0], - [(self.ts - 1) * 1000, 0.5], - [self.ts * 1000, 1.5]]) + end, + [ + [(self.ts - 3) * 1000, None], + [(self.ts - 2) * 1000, 1.0], + [(self.ts - 1) * 1000, 0.5], + [self.ts * 1000, 1.5], + ], + ) except AssertionError: self.assertEqual( - end, [[(self.ts - 2) * 1000, 1.0], - [(self.ts - 1) * 1000, 0.5], - [self.ts * 1000, 1.5], - [(self.ts + 1) * 1000, None]]) - - response = self.app.get(self.url, query_string={'target': 'test', - 'format': 'rickshaw'}) - data = json.loads(response.data.decode('utf-8')) - end = data[0]['datapoints'][-4:] + end, + [ + [(self.ts - 2) * 1000, 1.0], + [(self.ts - 1) * 1000, 0.5], + [self.ts * 1000, 1.5], + [(self.ts + 1) * 1000, None], + ], + ) + + response = self.app.get( + self.url, query_string={"target": "test", "format": "rickshaw"} + ) + data = json.loads(response.data.decode("utf-8")) + end = data[0]["datapoints"][-4:] try: self.assertEqual( - end, [{'x': self.ts - 3, 'y': None}, - {'x': self.ts - 2, 'y': 1.0}, - {'x': self.ts - 1, 'y': 0.5}, - {'x': self.ts, 'y': 1.5}]) + end, + [ + {"x": self.ts - 3, "y": None}, + {"x": self.ts - 2, "y": 1.0}, + {"x": self.ts - 1, "y": 0.5}, + {"x": self.ts, "y": 1.5}, + ], + ) except AssertionError: self.assertEqual( - end, [{'x': self.ts - 2, 'y': 1.0}, - {'x': self.ts - 1, 'y': 0.5}, - {'x': self.ts, 'y': 1.5}, - {'x': self.ts + 1, 'y': None}]) + end, + [ + {"x": self.ts - 2, "y": 1.0}, + {"x": self.ts - 1, "y": 0.5}, + {"x": self.ts, "y": 1.5}, + {"x": self.ts + 1, "y": None}, + ], + ) def test_render_constant_line(self): - response = self.app.get(self.url, query_string={ - 'target': 'constantLine(12)'}) - self.assertEqual(response.headers['Content-Type'], 'image/png') + response = self.app.get( + self.url, query_string={"target": "constantLine(12)"} + ) + self.assertEqual(response.headers["Content-Type"], "image/png") - response = self.app.get(self.url, query_string={ - 'target': 'constantLine(12)', 'format': 'json'}) - data = json.loads(response.data.decode('utf-8'))[0]['datapoints'] + response = self.app.get( + self.url, + query_string={"target": "constantLine(12)", "format": "json"}, + ) + data = json.loads(response.data.decode("utf-8"))[0]["datapoints"] self.assertEqual(len(data), 3) for point, _ts in data: self.assertEqual(point, 12) - response = self.app.get(self.url, query_string={ - 'target': 'constantLine(12)', 'format': 'json', - 'maxDataPoints': 12}) - data = json.loads(response.data.decode('utf-8'))[0]['datapoints'] + response = self.app.get( + self.url, + query_string={ + "target": "constantLine(12)", + "format": "json", + "maxDataPoints": 12, + }, + ) + data = json.loads(response.data.decode("utf-8"))[0]["datapoints"] self.assertEqual(len(data), 3) for point, _ts in data: self.assertEqual(point, 12) def test_float_maxdatapoints(self): - response = self.app.get(self.url, query_string={ - 'target': 'sin("foo")', 'format': 'json', - 'maxDataPoints': 5.5}) # rounded to int - data = json.loads(response.data.decode('utf-8'))[0]['datapoints'] + response = self.app.get( + self.url, + query_string={ + "target": 'sin("foo")', + "format": "json", + "maxDataPoints": 5.5, + }, + ) # rounded to int + data = json.loads(response.data.decode("utf-8"))[0]["datapoints"] self.assertEqual(len(data), 5) def test_constantline_pathexpr(self): - response = self.app.get(self.url, query_string={ - 'target': 'sumSeries(constantLine(12), constantLine(5))', - 'format': 'json', - }) - data = json.loads(response.data.decode('utf-8'))[0]['datapoints'] + response = self.app.get( + self.url, + query_string={ + "target": "sumSeries(constantLine(12), constantLine(5))", + "format": "json", + }, + ) + data = json.loads(response.data.decode("utf-8"))[0]["datapoints"] self.assertEqual([d[0] for d in data], [17, 17, 17]) def test_area_between(self): - response = self.app.get(self.url, query_string={ - 'target': ['areaBetween(sin("foo"), sin("bar", 2))'], - 'format': 'json', - }) - data = json.loads(response.data.decode('utf-8')) + response = self.app.get( + self.url, + query_string={ + "target": ['areaBetween(sin("foo"), sin("bar", 2))'], + "format": "json", + }, + ) + data = json.loads(response.data.decode("utf-8")) self.assertEqual(len(data), 2) def test_sumseries(self): - response = self.app.get(self.url, query_string={ - 'target': ['sumSeries(sin("foo"), sin("bar", 2))', - 'sin("baz", 3)'], - 'format': 'json', - }) - data = json.loads(response.data.decode('utf-8')) + response = self.app.get( + self.url, + query_string={ + "target": [ + 'sumSeries(sin("foo"), sin("bar", 2))', + 'sin("baz", 3)', + ], + "format": "json", + }, + ) + data = json.loads(response.data.decode("utf-8")) agg = {} for series in data: - agg[series['target']] = series['datapoints'] - for index, value in enumerate(agg['baz']): - self.assertEqual(value, agg['sumSeries(sin(bar),sin(foo))'][index]) - - response = self.app.get(self.url, query_string={ - 'target': ['sumSeries(sin("foo"), sin("bar", 2))', - 'sin("baz", 3)'], - 'format': 'json', - 'maxDataPoints': 100, - }) - data = json.loads(response.data.decode('utf-8')) + agg[series["target"]] = series["datapoints"] + for index, value in enumerate(agg["baz"]): + self.assertEqual( + value, agg["sumSeries(sin(bar),sin(foo))"][index] + ) + + response = self.app.get( + self.url, + query_string={ + "target": [ + 'sumSeries(sin("foo"), sin("bar", 2))', + 'sin("baz", 3)', + ], + "format": "json", + "maxDataPoints": 100, + }, + ) + data = json.loads(response.data.decode("utf-8")) agg = {} for series in data: - self.assertTrue(len(series['datapoints']) <= 100) - agg[series['target']] = series['datapoints'] - for index, value in enumerate(agg['baz']): - self.assertEqual(value, agg['sumSeries(sin(bar),sin(foo))'][index]) + self.assertTrue(len(series["datapoints"]) <= 100) + agg[series["target"]] = series["datapoints"] + for index, value in enumerate(agg["baz"]): + self.assertEqual( + value, agg["sumSeries(sin(bar),sin(foo))"][index] + ) def test_correct_timezone(self): - response = self.app.get(self.url, query_string={ - 'target': 'constantLine(12)', - 'format': 'json', - 'from': '07:00_20140226', - 'until': '08:00_20140226', - # tz is UTC - }) - data = json.loads(response.data.decode('utf-8'))[0]['datapoints'] + response = self.app.get( + self.url, + query_string={ + "target": "constantLine(12)", + "format": "json", + "from": "07:00_20140226", + "until": "08:00_20140226", + # tz is UTC + }, + ) + data = json.loads(response.data.decode("utf-8"))[0]["datapoints"] # all the from/until/tz combinations lead to the same window expected = [[12, 1393398000], [12, 1393399800], [12, 1393401600]] self.assertEqual(data, expected) - response = self.app.get(self.url, query_string={ - 'target': 'constantLine(12)', - 'format': 'json', - 'from': '08:00_20140226', - 'until': '09:00_20140226', - 'tz': 'Europe/Berlin', - }) - data = json.loads(response.data.decode('utf-8'))[0]['datapoints'] + response = self.app.get( + self.url, + query_string={ + "target": "constantLine(12)", + "format": "json", + "from": "08:00_20140226", + "until": "09:00_20140226", + "tz": "Europe/Berlin", + }, + ) + data = json.loads(response.data.decode("utf-8"))[0]["datapoints"] self.assertEqual(data, expected) def test_render_options(self): self.create_db() - db2 = os.path.join(WHISPER_DIR, 'foo.wsp') + db2 = os.path.join(WHISPER_DIR, "foo.wsp") whisper.create(db2, [(1, 60)]) ts = int(time.time()) whisper.update(db2, 0.5, ts - 2) for qs in [ - {'logBase': 'e'}, - {'logBase': 1}, - {'logBase': 0.5}, - {'logBase': 10}, - {'margin': -1}, - {'colorList': 'orange,green,blue,#0f00f0'}, - {'bgcolor': 'orange'}, - {'bgcolor': '000000'}, - {'bgcolor': '#000000'}, - {'bgcolor': '123456'}, - {'bgcolor': '#123456'}, - {'bgcolor': '#12345678'}, - {'bgcolor': 'aaabbb'}, - {'bgcolor': '#aaabbb'}, - {'bgcolor': '#aaabbbff'}, - {'fontBold': 'true'}, - {'title': 'Hellò'}, - {'title': 'true'}, - {'vtitle': 'Hellò'}, - {'title': 'Hellò', 'yAxisSide': 'right'}, - {'uniqueLegend': 'true', '_expr': 'secondYAxis({0})'}, - {'uniqueLegend': 'true', 'vtitleRight': 'foo', - '_expr': 'secondYAxis({0})'}, - {'rightWidth': '1', '_expr': 'secondYAxis({0})'}, - {'rightDashed': '1', '_expr': 'secondYAxis({0})'}, - {'rightColor': 'black', '_expr': 'secondYAxis({0})'}, - {'leftWidth': '1', 'target': ['secondYAxis(foo)', 'test']}, - {'leftDashed': '1', 'target': ['secondYAxis(foo)', 'test']}, - {'leftColor': 'black', 'target': ['secondYAxis(foo)', 'test']}, - {'width': '10', '_expr': 'secondYAxis({0})'}, - {'logBase': 'e', 'target': ['secondYAxis(foo)', 'test']}, - {'graphOnly': 'true', 'yUnitSystem': 'si'}, - {'graphOnly': 'true', 'yUnitSystem': 'wat'}, - {'lineMode': 'staircase'}, - {'lineMode': 'slope'}, - {'lineMode': 'slope', 'from': '-1s'}, - {'lineMode': 'connected'}, - {'min': 1, 'max': 2, 'thickness': 2, 'yUnitSystem': 'none'}, - {'yMax': 5, 'yLimit': 0.5, 'yStep': 0.1}, - {'yMax': 'max', 'yUnitSystem': 'binary'}, - {'yMaxLeft': 5, 'yLimitLeft': 0.5, 'yStepLeft': 0.1, - '_expr': 'secondYAxis({0})'}, - {'yMaxRight': 5, 'yLimitRight': 0.5, 'yStepRight': 0.1, - '_expr': 'secondYAxis({0})'}, - {'yMin': 0, 'yLimit': 0.5, 'yStep': 0.1}, - {'yMinLeft': 0, 'yLimitLeft': 0.5, 'yStepLeft': 0.1, - '_expr': 'secondYAxis({0})'}, - {'yMinRight': 0, 'yLimitRight': 0.5, 'yStepRight': 0.1, - '_expr': 'secondYAxis({0})'}, - {'areaMode': 'stacked', '_expr': 'stacked({0})'}, - {'lineMode': 'staircase', '_expr': 'stacked({0})'}, - {'areaMode': 'first', '_expr': 'stacked({0})'}, - {'areaMode': 'all', '_expr': 'stacked({0})'}, - {'areaMode': 'all', 'areaAlpha': 0.5, '_expr': 'secondYAxis({0})'}, - {'areaMode': 'all', 'areaAlpha': 0.5, - 'target': ['secondYAxis(foo)', 'test']}, - {'areaMode': 'stacked', 'areaAlpha': 0.5, '_expr': 'stacked({0})'}, - {'areaMode': 'stacked', 'areaAlpha': 'a', '_expr': 'stacked({0})'}, - {'areaMode': 'stacked', '_expr': 'drawAsInfinite({0})'}, - {'_expr': 'dashed(lineWidth({0}, 5))'}, - {'target': 'areaBetween(*)'}, - {'drawNullAsZero': 'true'}, - {'_expr': 'drawAsInfinite({0})'}, - {'graphType': 'pie', 'pieMode': 'average', 'title': 'Pie'}, - {'graphType': 'pie', 'pieMode': 'maximum', 'title': 'Pie'}, - {'graphType': 'pie', 'pieMode': 'minimum', 'title': 'Pie'}, - {'graphType': 'pie', 'pieMode': 'average', 'hideLegend': 'true'}, - {'graphType': 'pie', 'pieMode': 'average', 'valueLabels': 'none'}, - {'graphType': 'pie', 'pieMode': 'average', - 'valueLabels': 'number'}, - {'graphType': 'pie', 'pieMode': 'average', 'pieLabels': 'rotated'}, - {'graphType': 'pie', 'pieMode': 'average', 'areaAlpha': '0.1'}, - {'graphType': 'pie', 'pieMode': 'average', 'areaAlpha': 'none'}, - {'graphType': 'pie', 'pieMode': 'average', - 'valueLabelsColor': 'white'}, - {'noCache': 'true'}, - {'cacheTimeout': 5}, - {'cacheTimeout': 5}, # cache hit - {'tz': 'Europe/Berlin'}, + {"logBase": "e"}, + {"logBase": 1}, + {"logBase": 0.5}, + {"logBase": 10}, + {"margin": -1}, + {"colorList": "orange,green,blue,#0f00f0"}, + {"bgcolor": "orange"}, + {"bgcolor": "000000"}, + {"bgcolor": "#000000"}, + {"bgcolor": "123456"}, + {"bgcolor": "#123456"}, + {"bgcolor": "#12345678"}, + {"bgcolor": "aaabbb"}, + {"bgcolor": "#aaabbb"}, + {"bgcolor": "#aaabbbff"}, + {"fontBold": "true"}, + {"title": "Hellò"}, + {"title": "true"}, + {"vtitle": "Hellò"}, + {"title": "Hellò", "yAxisSide": "right"}, + {"uniqueLegend": "true", "_expr": "secondYAxis({0})"}, + { + "uniqueLegend": "true", + "vtitleRight": "foo", + "_expr": "secondYAxis({0})", + }, + {"rightWidth": "1", "_expr": "secondYAxis({0})"}, + {"rightDashed": "1", "_expr": "secondYAxis({0})"}, + {"rightColor": "black", "_expr": "secondYAxis({0})"}, + {"leftWidth": "1", "target": ["secondYAxis(foo)", "test"]}, + {"leftDashed": "1", "target": ["secondYAxis(foo)", "test"]}, + {"leftColor": "black", "target": ["secondYAxis(foo)", "test"]}, + {"width": "10", "_expr": "secondYAxis({0})"}, + {"logBase": "e", "target": ["secondYAxis(foo)", "test"]}, + {"graphOnly": "true", "yUnitSystem": "si"}, + {"graphOnly": "true", "yUnitSystem": "wat"}, + {"lineMode": "staircase"}, + {"lineMode": "slope"}, + {"lineMode": "slope", "from": "-1s"}, + {"lineMode": "connected"}, + {"min": 1, "max": 2, "thickness": 2, "yUnitSystem": "none"}, + {"yMax": 5, "yLimit": 0.5, "yStep": 0.1}, + {"yMax": "max", "yUnitSystem": "binary"}, + { + "yMaxLeft": 5, + "yLimitLeft": 0.5, + "yStepLeft": 0.1, + "_expr": "secondYAxis({0})", + }, + { + "yMaxRight": 5, + "yLimitRight": 0.5, + "yStepRight": 0.1, + "_expr": "secondYAxis({0})", + }, + {"yMin": 0, "yLimit": 0.5, "yStep": 0.1}, + { + "yMinLeft": 0, + "yLimitLeft": 0.5, + "yStepLeft": 0.1, + "_expr": "secondYAxis({0})", + }, + { + "yMinRight": 0, + "yLimitRight": 0.5, + "yStepRight": 0.1, + "_expr": "secondYAxis({0})", + }, + {"areaMode": "stacked", "_expr": "stacked({0})"}, + {"lineMode": "staircase", "_expr": "stacked({0})"}, + {"areaMode": "first", "_expr": "stacked({0})"}, + {"areaMode": "all", "_expr": "stacked({0})"}, + { + "areaMode": "all", + "areaAlpha": 0.5, + "_expr": "secondYAxis({0})", + }, + { + "areaMode": "all", + "areaAlpha": 0.5, + "target": ["secondYAxis(foo)", "test"], + }, + { + "areaMode": "stacked", + "areaAlpha": 0.5, + "_expr": "stacked({0})", + }, + { + "areaMode": "stacked", + "areaAlpha": "a", + "_expr": "stacked({0})", + }, + {"areaMode": "stacked", "_expr": "drawAsInfinite({0})"}, + {"_expr": "dashed(lineWidth({0}, 5))"}, + {"target": "areaBetween(*)"}, + {"drawNullAsZero": "true"}, + {"_expr": "drawAsInfinite({0})"}, + {"graphType": "pie", "pieMode": "average", "title": "Pie"}, + {"graphType": "pie", "pieMode": "maximum", "title": "Pie"}, + {"graphType": "pie", "pieMode": "minimum", "title": "Pie"}, + {"graphType": "pie", "pieMode": "average", "hideLegend": "true"}, + {"graphType": "pie", "pieMode": "average", "valueLabels": "none"}, + { + "graphType": "pie", + "pieMode": "average", + "valueLabels": "number", + }, + { + "graphType": "pie", + "pieMode": "average", + "pieLabels": "rotated", + }, + {"graphType": "pie", "pieMode": "average", "areaAlpha": "0.1"}, + {"graphType": "pie", "pieMode": "average", "areaAlpha": "none"}, + { + "graphType": "pie", + "pieMode": "average", + "valueLabelsColor": "white", + }, + {"noCache": "true"}, + {"cacheTimeout": 5}, + {"cacheTimeout": 5}, # cache hit + {"tz": "Europe/Berlin"}, ]: - if qs.setdefault('target', ['foo', 'test']) == ['foo', 'test']: - if '_expr' in qs: - expr = qs.pop('_expr') - qs['target'] = [expr.format(t) for t in qs['target']] + if qs.setdefault("target", ["foo", "test"]) == ["foo", "test"]: + if "_expr" in qs: + expr = qs.pop("_expr") + qs["target"] = [expr.format(t) for t in qs["target"]] response = self.app.get(self.url, query_string=qs) self.assertEqual(response.status_code, 200) - self.assertEqual(response.headers['Content-Type'], 'image/png') - if Cache is None or qs.get('noCache'): - self.assertEqual(response.headers['Pragma'], 'no-cache') - self.assertEqual(response.headers['Cache-Control'], 'no-cache') - self.assertFalse('Expires' in response.headers) + self.assertEqual(response.headers["Content-Type"], "image/png") + if Cache is None or qs.get("noCache"): + self.assertEqual(response.headers["Pragma"], "no-cache") + self.assertEqual( + response.headers["Cache-Control"], "no-cache" + ) + self.assertFalse("Expires" in response.headers) else: - self.assertEqual(response.headers['Cache-Control'], - 'max-age={0}'.format( - qs.get('cacheTimeout', 60))) - self.assertNotEqual(response.headers['Cache-Control'], - 'no-cache') - self.assertFalse('Pragma' in response.headers) - - for qs in [ - {'bgcolor': 'foo'}, - ]: - qs['target'] = 'test' + self.assertEqual( + response.headers["Cache-Control"], + "max-age={0}".format(qs.get("cacheTimeout", 60)), + ) + self.assertNotEqual( + response.headers["Cache-Control"], "no-cache" + ) + self.assertFalse("Pragma" in response.headers) + + for qs in [{"bgcolor": "foo"}]: + qs["target"] = "test" with self.assertRaises(ValueError): response = self.app.get(self.url, query_string=qs) - for qs in [ - {'lineMode': 'stacked'}, - ]: - qs['target'] = 'test' + for qs in [{"lineMode": "stacked"}]: + qs["target"] = "test" with self.assertRaises(AssertionError): response = self.app.get(self.url, query_string=qs) @@ -357,164 +516,231 @@ def test_render_validation(self): whisper.create(self.db, [(1, 60)]) response = self.app.get(self.url) - self.assertJSON(response, {'errors': { - 'target': 'This parameter is required.'}}, status_code=400) - - response = self.app.get(self.url, query_string={'graphType': 'foo', - 'target': 'test'}) - self.assertJSON(response, {'errors': { - 'graphType': "Invalid graphType 'foo', must be one of 'line', " - "'pie'."}}, status_code=400) - - response = self.app.get(self.url, query_string={'maxDataPoints': 'foo', - 'target': 'test'}) - self.assertJSON(response, {'errors': { - 'maxDataPoints': 'Must be an integer.'}}, status_code=400) - - response = self.app.get(self.url, query_string={ - 'from': '21:2020140313', - 'until': '21:2020140313', - 'target': 'test'}) - self.assertJSON(response, {'errors': { - 'from': 'Invalid empty time range', - 'until': 'Invalid empty time range', - }}, status_code=400) - - response = self.app.get(self.url, query_string={ - 'target': 'foo', - 'width': 100, - 'thickness': '1.5', - 'fontBold': 'true', - 'fontItalic': 'default', - }) + self.assertJSON( + response, + {"errors": {"target": "This parameter is required."}}, + status_code=400, + ) + + response = self.app.get( + self.url, query_string={"graphType": "foo", "target": "test"} + ) + self.assertJSON( + response, + { + "errors": { + "graphType": "Invalid graphType 'foo', must be one of " + "'line', 'pie'." + } + }, + status_code=400, + ) + + response = self.app.get( + self.url, query_string={"maxDataPoints": "foo", "target": "test"} + ) + self.assertJSON( + response, + {"errors": {"maxDataPoints": "Must be an integer."}}, + status_code=400, + ) + + response = self.app.get( + self.url, + query_string={ + "from": "21:2020140313", + "until": "21:2020140313", + "target": "test", + }, + ) + self.assertJSON( + response, + { + "errors": { + "from": "Invalid empty time range", + "until": "Invalid empty time range", + } + }, + status_code=400, + ) + + response = self.app.get( + self.url, + query_string={ + "target": "foo", + "width": 100, + "thickness": "1.5", + "fontBold": "true", + "fontItalic": "default", + }, + ) self.assertEqual(response.status_code, 200) - response = self.app.get(self.url, query_string={ - 'target': 'foo', 'tz': 'Europe/Lausanne'}) - self.assertJSON(response, {'errors': { - 'tz': "Unknown timezone: 'Europe/Lausanne'.", - }}, status_code=400) + response = self.app.get( + self.url, query_string={"target": "foo", "tz": "Europe/Lausanne"} + ) + self.assertJSON( + response, + {"errors": {"tz": "Unknown timezone: 'Europe/Lausanne'."}}, + status_code=400, + ) - response = self.app.get(self.url, query_string={'target': 'test:aa', - 'graphType': 'pie'}) - self.assertJSON(response, {'errors': { - 'target': "Invalid target: 'test:aa'.", - }}, status_code=400) + response = self.app.get( + self.url, query_string={"target": "test:aa", "graphType": "pie"} + ) + self.assertJSON( + response, + {"errors": {"target": "Invalid target: 'test:aa'."}}, + status_code=400, + ) - response = self.app.get(self.url, query_string={ - 'target': ['test', 'foo:1.2'], 'graphType': 'pie'}) + response = self.app.get( + self.url, + query_string={"target": ["test", "foo:1.2"], "graphType": "pie"}, + ) self.assertEqual(response.status_code, 200) - response = self.app.get(self.url, query_string={'target': ['test', - '']}) + response = self.app.get( + self.url, query_string={"target": ["test", ""]} + ) self.assertEqual(response.status_code, 200) - response = self.app.get(self.url, query_string={'target': 'test', - 'format': 'csv'}) - lines = response.data.decode('utf-8').strip().split('\n') + response = self.app.get( + self.url, query_string={"target": "test", "format": "csv"} + ) + lines = response.data.decode("utf-8").strip().split("\n") # 59 is a time race cond self.assertTrue(len(lines) in [59, 60]) - self.assertFalse(any([l.strip().split(',')[2] for l in lines])) + self.assertFalse(any([l.strip().split(",")[2] for l in lines])) - response = self.app.get(self.url, query_string={'target': 'test', - 'format': 'svg', - 'jsonp': 'foo'}) - jsonpsvg = response.data.decode('utf-8') + response = self.app.get( + self.url, + query_string={"target": "test", "format": "svg", "jsonp": "foo"}, + ) + jsonpsvg = response.data.decode("utf-8") self.assertTrue(jsonpsvg.startswith('foo("\\n")')) - response = self.app.get(self.url, query_string={'target': 'test', - 'format': 'svg'}) - svg = response.data.decode('utf-8') + response = self.app.get( + self.url, query_string={"target": "test", "format": "svg"} + ) + svg = response.data.decode("utf-8") self.assertTrue(svg.startswith('') + query = FindQuery("collectd", None, None) + self.assertEqual(repr(query), "") - query = FindQuery('collectd', start, None) - self.assertEqual(repr(query), '' - % time.ctime(start)) + query = FindQuery("collectd", start, None) + self.assertEqual( + repr(query), + "" % time.ctime(start), + ) - query = FindQuery('collectd', None, end) - self.assertEqual(repr(query), '' - % time.ctime(end)) + query = FindQuery("collectd", None, end) + self.assertEqual( + repr(query), + "" % time.ctime(end), + ) diff --git a/tox.ini b/tox.ini index 0974609..941bb10 100644 --- a/tox.ini +++ b/tox.ini @@ -116,11 +116,13 @@ deps = [testenv:lint] deps = + black flake8 flake8-import-order flake8-bugbear commands = - flake8 {toxinidir}/graphite_api {toxinidir}/tests + black graphite_api tests --check + flake8 {toxinidir} [testenv:docs] changedir = docs diff --git a/unittest_main.py b/unittest_main.py index b27718c..907a542 100644 --- a/unittest_main.py +++ b/unittest_main.py @@ -1,8 +1,10 @@ """Main entry point""" +import os.path import sys + if sys.argv[0].endswith("__main__.py"): - import os.path + # We change sys.argv[0] to make help message more useful # use executable without path, unquoted # (it's just a hint anyway) @@ -13,6 +15,6 @@ __unittest = True -from unittest.main import main, TestProgram +from unittest.main import main # noqa main(module=None)