From 62cbed68ec04d8aaa29163f03202f29a19a908de Mon Sep 17 00:00:00 2001 From: Hardy-Cooper Date: Wed, 18 Nov 2020 12:04:07 -0500 Subject: [PATCH 1/4] Initial commit: API call limiting --- cuckoo/common/config.py | 1 + cuckoo/private/cwd/conf/reporting.conf | 1 + cuckoo/reporting/jsondump.py | 67 ++++++++++++++++++++++---- 3 files changed, 60 insertions(+), 9 deletions(-) diff --git a/cuckoo/common/config.py b/cuckoo/common/config.py index 7728d513d6..c8dc26f093 100644 --- a/cuckoo/common/config.py +++ b/cuckoo/common/config.py @@ -741,6 +741,7 @@ class Config(object): "enabled": Boolean(True), "indent": Int(4), "calls": Boolean(True), + "call_limit": Int(0), }, "singlefile": { "enabled": Boolean(False), diff --git a/cuckoo/private/cwd/conf/reporting.conf b/cuckoo/private/cwd/conf/reporting.conf index 752d6ae24f..b10ea201a1 100644 --- a/cuckoo/private/cwd/conf/reporting.conf +++ b/cuckoo/private/cwd/conf/reporting.conf @@ -13,6 +13,7 @@ enabled = {{ reporting.feedback.enabled }} enabled = {{ reporting.jsondump.enabled }} indent = {{ reporting.jsondump.indent }} calls = {{ reporting.jsondump.calls }} +call_limit = {{ reporting.jsondump.call_limit }} [singlefile] # Enable creation of report.html and/or report.pdf? diff --git a/cuckoo/reporting/jsondump.py b/cuckoo/reporting/jsondump.py index 73041e7253..b9d6bcd6d4 100644 --- a/cuckoo/reporting/jsondump.py +++ b/cuckoo/reporting/jsondump.py @@ -11,6 +11,7 @@ from cuckoo.common.abstracts import Report from cuckoo.common.exceptions import CuckooReportError + def default(obj): if isinstance(obj, datetime.datetime): if obj.utcoffset() is not None: @@ -18,20 +19,62 @@ def default(obj): return calendar.timegm(obj.timetuple()) + obj.microsecond / 1000000.0 raise TypeError("%r is not JSON serializable" % obj) + class JsonDump(Report): """Save analysis results in JSON format.""" def erase_calls(self, results): """Temporarily remove calls from the report by replacing them with - empty lists.""" - if self.calls: - self.calls = None - return + empty lists. Or limiting calls that are made way too often.""" - self.calls = [] - for process in results.get("behavior", {}).get("processes", []): - self.calls.append(process["calls"]) - process["calls"] = [] + self.calls_to_be_restored = {} + if self.calls and self.call_limit: + # Create dict of {pid: {call that needs to be limited: current count, ...}, ...} + calls_to_be_limited = {} + behaviour = results.get("behavior", {}) + apistats = results.get("apistats", {}) + + # First fill calls_to_be_limited with... calls that need to be limited! + for pid in apistats: + calls_to_be_limited[pid] = {} + calls = apistats[pid] + for call in calls: + call_count = calls[call] + # If the number of calls according to apistats is too high, + # then start a counter + if call_count > self.call_limit: + calls_to_be_limited[pid][str(call_count)] = 0 + + # Now that we have the pid + api call to limit relationship, we can apply it to processes + for process in behaviour.get("processes", []): + pid = str(process["pid"]) + process_calls = process["calls"] + self.calls_to_be_restored[pid] = process_calls + + # If a process has no calls that need to be limited, move on + if pid not in calls_to_be_limited: + continue + + calls_to_be_limited_for_pid = calls_to_be_limited[pid] + limited_calls = [] + for call in process_calls: + api = str(call["api"]) + if api not in calls_to_be_limited_for_pid or calls_to_be_limited_for_pid[api] >= self.call_limit: + continue + calls_to_be_limited_for_pid[api] += 1 + limited_calls.append(call) + process["calls"] = limited_calls + return + elif self.calls and not self.call_limit: + # This means we want all calls, and don't need to restore anything + self.calls = False + return + else: + # This means we don't want calls in jsondump, therefore all calls need to be restored + for process in results.get("behavior", {}).get("processes", []): + pid = str(process["pid"]) + self.calls_to_be_restored[pid] = process["calls"] + process["calls"] = [] def restore_calls(self, results): """Restore calls that were temporarily removed in the report by @@ -40,7 +83,7 @@ def restore_calls(self, results): return for process in results.get("behavior", {}).get("processes", []): - process["calls"] = self.calls.pop(0) + process["calls"] = self.calls_to_be_restored.pop(str(process["pid"])) def run(self, results): """Writes report. @@ -54,6 +97,12 @@ def run(self, results): else: self.calls = self.options.get("calls", True) + if "json.call_limit" in self.task["options"]: + self.call_limit = int(self.task["options"]["json.call_limit"]) + else: + self.call_limit = self.options.get("call_limit", 0) + + # Attempting to write report without behaviour self.erase_calls(results) try: filepath = os.path.join(self.reports_path, "report.json") From d1718642cc4e649531480642cbf2db52a6929e1a Mon Sep 17 00:00:00 2001 From: Hardy-Cooper Date: Wed, 18 Nov 2020 13:49:13 -0500 Subject: [PATCH 2/4] Replacing with correct variable --- cuckoo/reporting/jsondump.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cuckoo/reporting/jsondump.py b/cuckoo/reporting/jsondump.py index b9d6bcd6d4..34391093e4 100644 --- a/cuckoo/reporting/jsondump.py +++ b/cuckoo/reporting/jsondump.py @@ -43,7 +43,7 @@ def erase_calls(self, results): # If the number of calls according to apistats is too high, # then start a counter if call_count > self.call_limit: - calls_to_be_limited[pid][str(call_count)] = 0 + calls_to_be_limited[pid][str(call)] = 0 # Now that we have the pid + api call to limit relationship, we can apply it to processes for process in behaviour.get("processes", []): From 5f08d82c8134f0bce6da82b8401cb0a875bdab48 Mon Sep 17 00:00:00 2001 From: Hardy-Cooper Date: Wed, 18 Nov 2020 14:52:11 -0500 Subject: [PATCH 3/4] Fixed a couple of bugs --- cuckoo/reporting/jsondump.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cuckoo/reporting/jsondump.py b/cuckoo/reporting/jsondump.py index 34391093e4..4019cfc3a3 100644 --- a/cuckoo/reporting/jsondump.py +++ b/cuckoo/reporting/jsondump.py @@ -32,7 +32,7 @@ def erase_calls(self, results): # Create dict of {pid: {call that needs to be limited: current count, ...}, ...} calls_to_be_limited = {} behaviour = results.get("behavior", {}) - apistats = results.get("apistats", {}) + apistats = behaviour.get("apistats", {}) # First fill calls_to_be_limited with... calls that need to be limited! for pid in apistats: @@ -42,7 +42,7 @@ def erase_calls(self, results): call_count = calls[call] # If the number of calls according to apistats is too high, # then start a counter - if call_count > self.call_limit: + if call_count >= self.call_limit: calls_to_be_limited[pid][str(call)] = 0 # Now that we have the pid + api call to limit relationship, we can apply it to processes From 0285f73e14d0c6f3f9b225afe640af7f33308045 Mon Sep 17 00:00:00 2001 From: Hardy-Cooper Date: Wed, 2 Dec 2020 10:53:42 -0500 Subject: [PATCH 4/4] Fixing logical error that was only adding calls that should be limited --- cuckoo/reporting/jsondump.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cuckoo/reporting/jsondump.py b/cuckoo/reporting/jsondump.py index 4019cfc3a3..cd708f89d4 100644 --- a/cuckoo/reporting/jsondump.py +++ b/cuckoo/reporting/jsondump.py @@ -52,16 +52,19 @@ def erase_calls(self, results): self.calls_to_be_restored[pid] = process_calls # If a process has no calls that need to be limited, move on - if pid not in calls_to_be_limited: + if pid not in calls_to_be_limited or not calls_to_be_limited.get(pid, {}): continue calls_to_be_limited_for_pid = calls_to_be_limited[pid] limited_calls = [] for call in process_calls: api = str(call["api"]) - if api not in calls_to_be_limited_for_pid or calls_to_be_limited_for_pid[api] >= self.call_limit: + # Skip call if over limit + if api in calls_to_be_limited_for_pid and calls_to_be_limited_for_pid[api] >= self.call_limit: continue - calls_to_be_limited_for_pid[api] += 1 + # Increment count if call is to be limited + elif api in calls_to_be_limited_for_pid: + calls_to_be_limited_for_pid[api] += 1 limited_calls.append(call) process["calls"] = limited_calls return