diff --git a/lib/cuckoo/common/cleaners_utils.py b/lib/cuckoo/common/cleaners_utils.py index bd1672b16a4..1342209c589 100644 --- a/lib/cuckoo/common/cleaners_utils.py +++ b/lib/cuckoo/common/cleaners_utils.py @@ -161,6 +161,8 @@ def free_space_monitor(path=False, return_value=False, processing=False, analysi cleanup_dict["delete_older_than"] = config.cleaner.analysis if config.cleaner.unused_files_in_mongodb: cleanup_dict["delete_unused_file_data_in_mongo"] = 1 + if config.cleaner.get("files"): + cleanup_dict["delete_files_items_older_than"] = config.cleaner.get("files") need_space, space_available = False, 0 # Calculate the free disk space in megabytes. @@ -455,7 +457,6 @@ def cuckoo_clean_failed_tasks(): # This need to init a console logger handler, because the standard # logger (init_logging()) logs to a file which will be deleted. create_structure() - # ToDo multi status tasks_list = db.list_tasks(status=f"{TASK_FAILED_ANALYSIS}|{TASK_FAILED_PROCESSING}|{TASK_FAILED_REPORTING}|{TASK_RECOVERED}") # ToDo rewrite for bulk delete @@ -810,6 +811,76 @@ def cape_clean_tlp(): delete_bulk_tasks_n_folders(tlp_tasks, False) +def files_clean_before(timerange: str): + """ + Clean up files in storage/files that are not referenced by any analysis + and are older than the specified time range. + """ + older_than = convert_into_time(timerange) + files_folder = os.path.join(CUCKOO_ROOT, "storage", "files") + analyses_folder = os.path.join(CUCKOO_ROOT, "storage", "analyses") + + if not path_exists(files_folder): + return + + # 1. Build set of referenced hashes from BOTH MongoDB (fast) and filesystem symlinks (absolute truth) + referenced = set() + + if is_reporting_db_connected() and repconf.mongodb.enabled and "mongo_find" in globals(): + try: + # Query all _id (SHA256) from files collection + cursor = mongo_find("files", {}, {"_id": 1}) + for doc in cursor: + referenced.add(doc["_id"]) + log.info("Loaded %d referenced files from MongoDB", len(referenced)) + except Exception as e: + log.error("Failed to query MongoDB for files: %s.", e) + + # Always scan the filesystem to verify actual active symlinks (safety gate to prevent broken symlinks) + if path_exists(analyses_folder): + log.info("Scanning analysis folders for file references...") + with os.scandir(analyses_folder) as it: + for entry in it: + if not entry.is_dir(): + continue + for subdir in ("selfextracted", "files", "CAPE", "procdump"): + check_dir = os.path.join(entry.path, subdir) + if path_exists(check_dir): + with os.scandir(check_dir) as se_it: + for se_entry in se_it: + if se_entry.is_symlink(): + try: + target = os.readlink(se_entry.path) + # Check if it points to storage/files using realpath + if os.path.realpath(target).startswith(os.path.realpath(files_folder)): + referenced.add(os.path.basename(target)) + except OSError: + pass + + # 2. Iterate storage/files and clean + for root, _, filenames in os.walk(files_folder, topdown=False): + for sha256 in filenames: + if sha256 in referenced: + continue + + file_path = os.path.join(root, sha256) + try: + st_ctime = path_get_date(file_path) + # Correct logic: delete if OLDER than limit (<) + if datetime.fromtimestamp(st_ctime) < older_than: + path_delete(file_path) + except Exception as e: + log.warning("Error checking/deleting file %s: %s", file_path, e) + + # Try to remove empty directories (except the root files_folder) + if root != files_folder: + try: + os.rmdir(root) + except OSError: + # Directory not empty or other error + pass + + def binaries_clean_before(timerange: str): # In case if "delete_bin_copy = off" we might need to clean binaries # find storage/binaries/ -name "*" -type f -mtime 5 -delete @@ -917,11 +988,14 @@ def execute_cleanup(args: dict, init_log=True): if args.get("delete_tmp_items_older_than"): tmp_clean_before(args["delete_tmp_items_older_than"]) + if args.get("delete_unused_file_data_in_mongo"): + delete_unused_file_data_in_mongo() + if args.get("delete_binaries_items_older_than"): binaries_clean_before(args["delete_binaries_items_older_than"]) - if args.get("delete_unused_file_data_in_mongo"): - delete_unused_file_data_in_mongo() + if args.get("delete_files_items_older_than"): + files_clean_before(args["delete_files_items_older_than"]) if args.get("cleanup_files_collection_by_id"): cleanup_files_collection_by_id(args["cleanup_files_collection_by_id"]) diff --git a/lib/cuckoo/common/integrations/file_extra_info.py b/lib/cuckoo/common/integrations/file_extra_info.py index b324623b204..a119cecdef7 100644 --- a/lib/cuckoo/common/integrations/file_extra_info.py +++ b/lib/cuckoo/common/integrations/file_extra_info.py @@ -43,8 +43,9 @@ path_mkdir, path_read_file, path_write_file, + path_delete, ) -from lib.cuckoo.common.utils import get_options, is_text_file +from lib.cuckoo.common.utils import get_files_storage_path, get_options, is_text_file try: from sflock import unpack @@ -394,22 +395,41 @@ def _extracted_files_metadata( file_info["path"] = dest_path file_info["guest_paths"] = [file_info["name"]] file_info["name"] = os.path.basename(dest_path) + # Define the new central storage for all files (extracted, dropped, etc.) + master_file_path = get_files_storage_path(file_info["sha256"]) + files_storage_dir = os.path.dirname(master_file_path) + + # 1. Ensure file is in central storage + if not path_exists(master_file_path): + path_mkdir(files_storage_dir, exist_ok=True) + shutil.move(full_path, master_file_path) + elif path_exists(full_path): + # We already have it, delete the temp duplicate + path_delete(full_path) + + # 2. Create symlink in analysis folder (or copy if link fails) if not path_exists(dest_path): - shutil.move(full_path, dest_path) - print( - json.dumps( - { - "path": os.path.join("files", file_info["sha256"]), - "filepath": file_info["name"], - "pids": [], - "ppids": [], - "metadata": "", - "category": "files", - }, - ensure_ascii=False, - ), - file=f, - ) + try: + os.symlink(master_file_path, dest_path) + except OSError: + # Fallback to copy on error + shutil.copy(master_file_path, dest_path) + + # Update files.json for UI/Reporting to correctly reference the symlinked file + print( + json.dumps( + { + "path": file_info["sha256"], # Store just the SHA256 + "filepath": file_info["name"], + "pids": [], + "ppids": [], + "metadata": "", + "category": "selfextracted", + }, + ensure_ascii=False, + ), + file=f, + ) file_info["data"] = is_text_file(file_info, destination_folder, processing_conf.CAPE.buffer) metadata.append(file_info) diff --git a/lib/cuckoo/common/path_utils.py b/lib/cuckoo/common/path_utils.py index b0d72da0be2..408092579da 100644 --- a/lib/cuckoo/common/path_utils.py +++ b/lib/cuckoo/common/path_utils.py @@ -37,9 +37,12 @@ def path_safe(path: str) -> bool: def path_exists(path: str, windows: bool = False) -> bool: - if not windows: - return Path(path_to_ascii(path)).exists() - return PureWindowsPath(path_to_ascii(path)).exists() + try: + if not windows: + return Path(path_to_ascii(path)).exists() + return PureWindowsPath(path_to_ascii(path)).exists() + except (OSError, ValueError): + return False def path_get_size(path: str): @@ -51,11 +54,17 @@ def path_get_date(path: str, value: str = "st_ctime"): def path_is_file(path: str) -> bool: - return Path(path_to_ascii(path)).is_file() + try: + return Path(path_to_ascii(path)).is_file() + except (OSError, ValueError): + return False def path_is_dir(path: str) -> bool: - return Path(path_to_ascii(path)).is_dir() + try: + return Path(path_to_ascii(path)).is_dir() + except (OSError, ValueError): + return False def path_read_file(path: str, mode="bytes"): diff --git a/lib/cuckoo/common/utils.py b/lib/cuckoo/common/utils.py index 69f52adfb60..10d93c4fe2b 100644 --- a/lib/cuckoo/common/utils.py +++ b/lib/cuckoo/common/utils.py @@ -199,6 +199,20 @@ def get_memdump_path(memdump_id, analysis_folder=False): ) +def get_files_storage_path(sha256: str) -> str: + """ + Get the path to the storage/files directory for a given SHA256. + Uses sharding (e.g., storage/files/ab/cd/abcdef...) to avoid + too many files in a single directory. + If the file exists in the flat legacy path (storage/files/abcdef...), + returns that path for backward compatibility. + """ + flat_path = os.path.join(CUCKOO_ROOT, "storage", "files", sha256) + if os.path.exists(flat_path): + return flat_path + return os.path.join(CUCKOO_ROOT, "storage", "files", sha256[:2], sha256[2:4], sha256) + + def validate_referrer(url): if not url: return None diff --git a/modules/processing/CAPE.py b/modules/processing/CAPE.py index f89b98b867c..08f75d9187e 100644 --- a/modules/processing/CAPE.py +++ b/modules/processing/CAPE.py @@ -17,6 +17,7 @@ import json import logging import os +import shutil import timeit from contextlib import suppress from pathlib import Path @@ -26,12 +27,13 @@ from lib.cuckoo.common.config import Config from lib.cuckoo.common.integrations.file_extra_info import DuplicatesType, static_file_info from lib.cuckoo.common.objects import File -from lib.cuckoo.common.path_utils import path_exists +from lib.cuckoo.common.path_utils import path_exists, path_mkdir from lib.cuckoo.common.replace_patterns_utils import _clean_path from lib.cuckoo.common.utils import ( add_family_detection, convert_to_printable_and_truncate, get_clamav_consensus, + get_files_storage_path, get_options, make_bytes, texttypes, @@ -184,6 +186,29 @@ def process_file(self, file_path, append_file, metadata: dict, *, category: str, f = File(file_path, metadata.get("metadata", "")) sha256 = f.get_sha256() + # Deduplicate dropped, procdump, CAPE, and package files to storage/files + if category in ("dropped", "procdump", "CAPE", "package", "procmemory") and not os.path.islink(file_path): + try: + master_path = get_files_storage_path(sha256) + files_storage_dir = os.path.dirname(master_path) + + if not path_exists(master_path): + path_mkdir(files_storage_dir, exist_ok=True) + # Move file + shutil.move(file_path, master_path) + else: + # Already exists, delete duplicate + os.remove(file_path) + + # Link back + try: + os.symlink(master_path, file_path) + except (OSError, AttributeError): + shutil.copy(master_path, file_path) + + except Exception as e: + log.error("Deduplication failed for %s: %s", file_path, e) + if sha256 in duplicated["sha256"]: log.debug("Skipping file that has already been processed: %s", sha256) return