feat(storage): Introduce sharded files directories, deduplication, and NFS sync utilities

lib/cuckoo/common/cleaners_utils.py

@@ -161,6 +161,8 @@ def free_space_monitor(path=False, return_value=False, processing=False, analysi
         cleanup_dict["delete_older_than"] = config.cleaner.analysis
     if config.cleaner.unused_files_in_mongodb:
         cleanup_dict["delete_unused_file_data_in_mongo"] = 1
+    if config.cleaner.get("files"):
+        cleanup_dict["delete_files_items_older_than"] = config.cleaner.get("files")
 
     need_space, space_available = False, 0
     # Calculate the free disk space in megabytes.
@@ -455,7 +457,6 @@ def cuckoo_clean_failed_tasks():
     # This need to init a console logger handler, because the standard
     # logger (init_logging()) logs to a file which will be deleted.
     create_structure()
-
     # ToDo multi status
     tasks_list = db.list_tasks(status=f"{TASK_FAILED_ANALYSIS}|{TASK_FAILED_PROCESSING}|{TASK_FAILED_REPORTING}|{TASK_RECOVERED}")
     # ToDo rewrite for bulk delete
@@ -810,6 +811,76 @@ def cape_clean_tlp():
     delete_bulk_tasks_n_folders(tlp_tasks, False)
 
 
+def files_clean_before(timerange: str):
+    """
+    Clean up files in storage/files that are not referenced by any analysis
+    and are older than the specified time range.
+    """
+    older_than = convert_into_time(timerange)
+    files_folder = os.path.join(CUCKOO_ROOT, "storage", "files")
+    analyses_folder = os.path.join(CUCKOO_ROOT, "storage", "analyses")
+
+    if not path_exists(files_folder):
+        return
+
+    # 1. Build set of referenced hashes from BOTH MongoDB (fast) and filesystem symlinks (absolute truth)
+    referenced = set()
+
+    if is_reporting_db_connected() and repconf.mongodb.enabled and "mongo_find" in globals():
+        try:
+            # Query all _id (SHA256) from files collection
+            cursor = mongo_find("files", {}, {"_id": 1})
+            for doc in cursor:
+                referenced.add(doc["_id"])
+            log.info("Loaded %d referenced files from MongoDB", len(referenced))
+        except Exception as e:
+            log.error("Failed to query MongoDB for files: %s.", e)
+
+    # Always scan the filesystem to verify actual active symlinks (safety gate to prevent broken symlinks)
+    if path_exists(analyses_folder):
+        log.info("Scanning analysis folders for file references...")
+        with os.scandir(analyses_folder) as it:
+            for entry in it:
+                if not entry.is_dir():
+                    continue
+                for subdir in ("selfextracted", "files", "CAPE", "procdump"):
+                    check_dir = os.path.join(entry.path, subdir)
+                    if path_exists(check_dir):
+                        with os.scandir(check_dir) as se_it:
+                            for se_entry in se_it:
+                                if se_entry.is_symlink():
+                                    try:
+                                        target = os.readlink(se_entry.path)
+                                        # Check if it points to storage/files using realpath
+                                        if os.path.realpath(target).startswith(os.path.realpath(files_folder)):
+                                            referenced.add(os.path.basename(target))
+                                    except OSError:
+                                        pass
+
+    # 2. Iterate storage/files and clean
+    for root, _, filenames in os.walk(files_folder, topdown=False):
+        for sha256 in filenames:
+            if sha256 in referenced:
+                continue
+
+            file_path = os.path.join(root, sha256)
+            try:
+                st_ctime = path_get_date(file_path)
+                # Correct logic: delete if OLDER than limit (<)
+                if datetime.fromtimestamp(st_ctime) < older_than:
+                    path_delete(file_path)
+            except Exception as e:
+                log.warning("Error checking/deleting file %s: %s", file_path, e)
+
+        # Try to remove empty directories (except the root files_folder)
+        if root != files_folder:
+            try:
+                os.rmdir(root)
+            except OSError:
+                # Directory not empty or other error
+                pass
+
+
 def binaries_clean_before(timerange: str):
     # In case if "delete_bin_copy = off" we might need to clean binaries
     # find storage/binaries/ -name "*" -type f -mtime 5 -delete
@@ -917,11 +988,14 @@ def execute_cleanup(args: dict, init_log=True):
     if args.get("delete_tmp_items_older_than"):
         tmp_clean_before(args["delete_tmp_items_older_than"])
 
+    if args.get("delete_unused_file_data_in_mongo"):
+        delete_unused_file_data_in_mongo()
+
     if args.get("delete_binaries_items_older_than"):
         binaries_clean_before(args["delete_binaries_items_older_than"])
 
-    if args.get("delete_unused_file_data_in_mongo"):
-        delete_unused_file_data_in_mongo()
+    if args.get("delete_files_items_older_than"):
+        files_clean_before(args["delete_files_items_older_than"])
 
     if args.get("cleanup_files_collection_by_id"):
         cleanup_files_collection_by_id(args["cleanup_files_collection_by_id"])

lib/cuckoo/common/integrations/file_extra_info.py

@@ -43,8 +43,9 @@
     path_mkdir,
     path_read_file,
     path_write_file,
+    path_delete,
 )
-from lib.cuckoo.common.utils import get_options, is_text_file
+from lib.cuckoo.common.utils import get_files_storage_path, get_options, is_text_file
 
 try:
     from sflock import unpack
@@ -394,22 +395,41 @@ def _extracted_files_metadata(
             file_info["path"] = dest_path
             file_info["guest_paths"] = [file_info["name"]]
             file_info["name"] = os.path.basename(dest_path)
+            # Define the new central storage for all files (extracted, dropped, etc.)
+            master_file_path = get_files_storage_path(file_info["sha256"])
+            files_storage_dir = os.path.dirname(master_file_path)
+
+            # 1. Ensure file is in central storage
+            if not path_exists(master_file_path):
+                path_mkdir(files_storage_dir, exist_ok=True)
+                shutil.move(full_path, master_file_path)
+            elif path_exists(full_path):
+                # We already have it, delete the temp duplicate
+                path_delete(full_path)
+
+            # 2. Create symlink in analysis folder (or copy if link fails)
             if not path_exists(dest_path):
-                shutil.move(full_path, dest_path)
-                print(
-                    json.dumps(
-                        {
-                            "path": os.path.join("files", file_info["sha256"]),
-                            "filepath": file_info["name"],
-                            "pids": [],
-                            "ppids": [],
-                            "metadata": "",
-                            "category": "files",
-                        },
-                        ensure_ascii=False,
-                    ),
-                    file=f,
-                )
+                try:
+                    os.symlink(master_file_path, dest_path)
+                except OSError:
+                    # Fallback to copy on error
+                    shutil.copy(master_file_path, dest_path)
+
+            # Update files.json for UI/Reporting to correctly reference the symlinked file
+            print(
+                json.dumps(
+                    {
+                        "path": file_info["sha256"],  # Store just the SHA256
+                        "filepath": file_info["name"],
+                        "pids": [],
+                        "ppids": [],
+                        "metadata": "",
+                        "category": "selfextracted",
+                    },
+                    ensure_ascii=False,
+                ),
+                file=f,
+            )
             file_info["data"] = is_text_file(file_info, destination_folder, processing_conf.CAPE.buffer)
             metadata.append(file_info)
 

lib/cuckoo/common/path_utils.py

@@ -37,9 +37,12 @@ def path_safe(path: str) -> bool:
 
 
 def path_exists(path: str, windows: bool = False) -> bool:
-    if not windows:
-        return Path(path_to_ascii(path)).exists()
-    return PureWindowsPath(path_to_ascii(path)).exists()
+    try:
+        if not windows:
+            return Path(path_to_ascii(path)).exists()
+        return PureWindowsPath(path_to_ascii(path)).exists()
+    except (OSError, ValueError):
+        return False
 
 
 def path_get_size(path: str):
@@ -51,11 +54,17 @@ def path_get_date(path: str, value: str = "st_ctime"):
 
 
 def path_is_file(path: str) -> bool:
-    return Path(path_to_ascii(path)).is_file()
+    try:
+        return Path(path_to_ascii(path)).is_file()
+    except (OSError, ValueError):
+        return False
 
 
 def path_is_dir(path: str) -> bool:
-    return Path(path_to_ascii(path)).is_dir()
+    try:
+        return Path(path_to_ascii(path)).is_dir()
+    except (OSError, ValueError):
+        return False
 
 
 def path_read_file(path: str, mode="bytes"):

lib/cuckoo/common/utils.py

@@ -199,6 +199,20 @@ def get_memdump_path(memdump_id, analysis_folder=False):
     )
 
 
+def get_files_storage_path(sha256: str) -> str:
+    """
+    Get the path to the storage/files directory for a given SHA256.
+    Uses sharding (e.g., storage/files/ab/cd/abcdef...) to avoid
+    too many files in a single directory.
+    If the file exists in the flat legacy path (storage/files/abcdef...),
+    returns that path for backward compatibility.
+    """
+    flat_path = os.path.join(CUCKOO_ROOT, "storage", "files", sha256)
+    if os.path.exists(flat_path):
+        return flat_path
+    return os.path.join(CUCKOO_ROOT, "storage", "files", sha256[:2], sha256[2:4], sha256)
+
+
 def validate_referrer(url):
     if not url:
         return None

modules/processing/CAPE.py

@@ -17,6 +17,7 @@
 import json
 import logging
 import os
+import shutil
 import timeit
 from contextlib import suppress
 from pathlib import Path
@@ -26,12 +27,13 @@
 from lib.cuckoo.common.config import Config
 from lib.cuckoo.common.integrations.file_extra_info import DuplicatesType, static_file_info
 from lib.cuckoo.common.objects import File
-from lib.cuckoo.common.path_utils import path_exists
+from lib.cuckoo.common.path_utils import path_exists, path_mkdir
 from lib.cuckoo.common.replace_patterns_utils import _clean_path
 from lib.cuckoo.common.utils import (
     add_family_detection,
     convert_to_printable_and_truncate,
     get_clamav_consensus,
+    get_files_storage_path,
     get_options,
     make_bytes,
     texttypes,
@@ -184,6 +186,29 @@ def process_file(self, file_path, append_file, metadata: dict, *, category: str,
         f = File(file_path, metadata.get("metadata", ""))
         sha256 = f.get_sha256()
 
+        # Deduplicate dropped, procdump, CAPE, and package files to storage/files
+        if category in ("dropped", "procdump", "CAPE", "package", "procmemory") and not os.path.islink(file_path):
+            try:
+                master_path = get_files_storage_path(sha256)
+                files_storage_dir = os.path.dirname(master_path)
+
+                if not path_exists(master_path):
+                    path_mkdir(files_storage_dir, exist_ok=True)
+                    # Move file
+                    shutil.move(file_path, master_path)
+                else:
+                    # Already exists, delete duplicate
+                    os.remove(file_path)
+
+                # Link back
+                try:
+                    os.symlink(master_path, file_path)
+                except (OSError, AttributeError):
+                    shutil.copy(master_path, file_path)
+
+            except Exception as e:
+                log.error("Deduplication failed for %s: %s", file_path, e)
+
         if sha256 in duplicated["sha256"]:
             log.debug("Skipping file that has already been processed: %s", sha256)
             return