diff --git a/migrations/001_consolidation.sql b/migrations/001_consolidation.sql index 71e92ea..a40ae2a 100644 --- a/migrations/001_consolidation.sql +++ b/migrations/001_consolidation.sql @@ -40,10 +40,12 @@ CREATE TABLE IF NOT EXISTS arkham_vectors.collections ( ); -- Main embeddings table +-- Using untyped vector allows variable dimensions per collection/model +-- Each collection tracks its own vector_size in arkham_vectors.collections CREATE TABLE IF NOT EXISTS arkham_vectors.embeddings ( id VARCHAR(36) PRIMARY KEY, collection VARCHAR(100) NOT NULL, - embedding vector(2000), -- Max pgvector dimension with index + embedding vector, -- Untyped: supports 384 (MiniLM), 768 (mpnet), 1024 (bge-m3), etc. payload JSONB DEFAULT '{}', created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP diff --git a/packages/arkham-frame/arkham_frame/services/documents.py b/packages/arkham-frame/arkham_frame/services/documents.py index 0ede91d..63d9d85 100644 --- a/packages/arkham-frame/arkham_frame/services/documents.py +++ b/packages/arkham-frame/arkham_frame/services/documents.py @@ -37,6 +37,7 @@ class DocumentStatus(str, Enum): COMPLETED = "completed" FAILED = "failed" ARCHIVED = "archived" + MERGED = "merged" # Document merged into another @dataclass @@ -897,26 +898,32 @@ async def search( if filters: vector_filter.update(filters) - # Perform vector search - results = await self.vectors.search( - collection="documents", - query_text=query, + # Perform vector search using text query + # search_text handles embedding internally + results = await self.vectors.search_text( + collection="arkham_documents", # Use correct collection name + text=query, limit=limit, filter=vector_filter if vector_filter else None, ) search_results = [] for result in results: - doc_id = result.get("document_id") + # SearchResult is a dataclass with .payload attribute + payload = result.payload if hasattr(result, 'payload') else result + doc_id = payload.get("document_id") if isinstance(payload, dict) else None if doc_id: doc = await self.get_document(doc_id) if doc: + # Extract score from SearchResult dataclass + score = result.score if hasattr(result, 'score') else (payload.get("score", 0.0) if isinstance(payload, dict) else 0.0) + highlights = payload.get("highlights", []) if isinstance(payload, dict) else [] search_results.append( SearchResult( document=doc, chunk=None, # Could load chunk if needed - score=result.get("score", 0.0), - highlights=result.get("highlights", []), + score=score, + highlights=highlights, ) ) diff --git a/packages/arkham-shard-anomalies/arkham_shard_anomalies/api.py b/packages/arkham-shard-anomalies/arkham_shard_anomalies/api.py index 072aa9b..e9091e9 100644 --- a/packages/arkham-shard-anomalies/arkham_shard_anomalies/api.py +++ b/packages/arkham-shard-anomalies/arkham_shard_anomalies/api.py @@ -24,6 +24,11 @@ AnomalyStats, StatusUpdate, AnalystNote, + # Hidden content models + HiddenContentConfig, + HiddenContentScan, + HiddenContentScanType, + HiddenContentStats, ) logger = logging.getLogger(__name__) @@ -32,20 +37,24 @@ # These get set by the shard on initialization _detector = None +_hidden_detector = None _store = None _event_bus = None _db = None _vectors = None +_storage = None -def init_api(detector, store, event_bus, db=None, vectors=None): +def init_api(detector, store, event_bus, db=None, vectors=None, hidden_detector=None, storage=None): """Initialize API with shard dependencies.""" - global _detector, _store, _event_bus, _db, _vectors + global _detector, _hidden_detector, _store, _event_bus, _db, _vectors, _storage _detector = detector + _hidden_detector = hidden_detector _store = store _event_bus = event_bus _db = db _vectors = vectors + _storage = storage def get_shard(request: Request) -> "AnomaliesShard": @@ -978,18 +987,20 @@ async def _detect_content_anomalies(doc_id: str, text: str) -> List[Anomaly]: # 2. Get embeddings for corpus # 3. Call _detector.detect_content_anomalies() - # Check if vector search is available - if hasattr(_vectors, 'search'): - # Search for similar documents - results = await _vectors.search( - collection="documents", - query=text[:1000], # Use first 1000 chars as query + # Check if vector text search is available (embeds text then searches) + if hasattr(_vectors, 'search_text'): + # Search for similar documents using text query + # search_text handles embedding internally + results = await _vectors.search_text( + collection="arkham_documents", # Use correct collection name + text=text[:1000], # Use first 1000 chars as query limit=10, ) # If this document is very different from results, it might be anomalous if results and len(results) > 0: - avg_score = sum(r.get('score', 0) for r in results) / len(results) + # SearchResult is a dataclass with .score attribute + avg_score = sum(r.score if hasattr(r, 'score') else r.get('score', 0) for r in results) / len(results) if avg_score < 0.3: # Low similarity to corpus import uuid from datetime import datetime @@ -1113,3 +1124,386 @@ async def ai_junior_analyst(request: Request, body: AIJuniorAnalystRequest): "X-Accel-Buffering": "no", }, ) + + +# ============================================================================= +# Hidden Content Detection Endpoints +# ============================================================================= + + +class HiddenContentScanRequest(BaseModel): + """Request to scan document for hidden content.""" + doc_id: str + scan_type: str = "stego" # entropy, lsb, magic, stego (full) + config: dict = {} + + +class HiddenContentQuickScanRequest(BaseModel): + """Request for quick entropy-only scan.""" + doc_ids: list[str] + + +class HiddenContentScanResponse(BaseModel): + """Response from hidden content scan.""" + scan: dict + anomaly_created: bool = False + + +class HiddenContentListResponse(BaseModel): + """Paginated list of hidden content scans.""" + total: int + items: list[dict] + offset: int + limit: int + + +class HiddenContentStatsResponse(BaseModel): + """Hidden content detection statistics.""" + stats: dict + + +@router.post("/hidden-content/scan", response_model=HiddenContentScanResponse) +async def scan_hidden_content( + request: Request, + body: HiddenContentScanRequest, +): + """ + Scan a document for hidden content. + + Performs steganography and hidden data detection: + - Entropy analysis (detects encrypted/compressed data) + - LSB pattern analysis (detects image steganography) + - File type mismatch detection + - Histogram analysis for images + + Args: + body: Scan request with doc_id and options + + Returns: + Scan results with findings + """ + shard = get_shard(request) + + if not shard.hidden_detector: + raise HTTPException( + status_code=503, + detail="Hidden content detector not available" + ) + + # Get document metadata and file path + if not _db: + raise HTTPException(status_code=503, detail="Database not available") + + doc_row = await _db.fetch_one( + """SELECT id, filename, storage_id, mime_type, file_size, metadata + FROM arkham_frame.documents WHERE id = :doc_id""", + {"doc_id": body.doc_id} + ) + + if not doc_row: + raise HTTPException(status_code=404, detail=f"Document {body.doc_id} not found") + + # Get storage path from storage_id or metadata + storage_id = doc_row.get("storage_id") + metadata = doc_row.get("metadata") or {} + if isinstance(metadata, str): + import json + try: + metadata = json.loads(metadata) + except json.JSONDecodeError: + metadata = {} + + storage_path = metadata.get("storage_path") + if not storage_id and not storage_path: + raise HTTPException( + status_code=400, + detail="Document has no associated file storage" + ) + + # Read file content using storage service + try: + if _storage and storage_id: + file_data = (await _storage.retrieve(storage_id))[0] + elif storage_path: + from pathlib import Path + file_data = Path(storage_path).read_bytes() + else: + raise ValueError("No storage path available") + except Exception as e: + raise HTTPException( + status_code=500, + detail=f"Failed to read file: {e}" + ) + + # Get file extension + filename = doc_row.get("filename", "") + file_ext = "." + filename.rsplit(".", 1)[-1] if "." in filename else "" + mime_type = doc_row.get("mime_type", "") + + # Configure detector if custom config provided + if body.config: + config = HiddenContentConfig(**body.config) + shard.hidden_detector.config = config + + # Perform scan (use storage_path for file path reference) + file_path = storage_path or storage_id + scan_result = shard.hidden_detector.full_scan( + doc_id=body.doc_id, + file_path=file_path, + file_data=file_data, + file_extension=file_ext, + mime_type=mime_type, + ) + + # Store the scan result + await shard._store_hidden_content_scan(scan_result) + + # Create anomaly if significant findings + anomaly_created = False + if scan_result.stego_confidence >= 0.7 or scan_result.file_mismatch: + import uuid + from datetime import datetime + from .models import AnomalyType, SeverityLevel, AnomalyStatus + + severity = SeverityLevel.HIGH if scan_result.stego_confidence >= 0.8 else SeverityLevel.MEDIUM + + anomaly = Anomaly( + id=str(uuid.uuid4()), + doc_id=body.doc_id, + anomaly_type=AnomalyType.HIDDEN_CONTENT, + status=AnomalyStatus.DETECTED, + score=scan_result.stego_confidence, + severity=severity, + confidence=scan_result.stego_confidence, + explanation="; ".join(scan_result.findings) if scan_result.findings else "Hidden content detected", + details={ + "scan_id": scan_result.id, + "indicators": len(scan_result.stego_indicators), + "file_mismatch": scan_result.file_mismatch, + "entropy_global": scan_result.entropy_global, + }, + detected_at=datetime.utcnow(), + ) + + await _store.create_anomaly(anomaly) + scan_result.anomaly_created = True + anomaly_created = True + + # Emit event + if _event_bus: + await _event_bus.emit( + "anomalies.hidden_content.detected", + { + "doc_id": body.doc_id, + "scan_id": scan_result.id, + "confidence": scan_result.stego_confidence, + "findings_count": len(scan_result.findings), + }, + source="anomalies-shard", + ) + + # Convert to response dict + scan_dict = { + "id": scan_result.id, + "doc_id": scan_result.doc_id, + "scan_type": scan_result.scan_type.value, + "scan_status": scan_result.scan_status.value, + "entropy_global": scan_result.entropy_global, + "entropy_regions": [ + { + "start_offset": r.start_offset, + "end_offset": r.end_offset, + "entropy_value": r.entropy_value, + "is_anomalous": r.is_anomalous, + "description": r.description, + } + for r in scan_result.entropy_regions + ], + "magic_expected": scan_result.magic_expected, + "magic_actual": scan_result.magic_actual, + "file_mismatch": scan_result.file_mismatch, + "lsb_result": { + "bit_ratio": scan_result.lsb_result.bit_ratio, + "chi_square_value": scan_result.lsb_result.chi_square_value, + "chi_square_p_value": scan_result.lsb_result.chi_square_p_value, + "is_suspicious": scan_result.lsb_result.is_suspicious, + "confidence": scan_result.lsb_result.confidence, + "sample_size": scan_result.lsb_result.sample_size, + } if scan_result.lsb_result else None, + "stego_indicators": [ + { + "indicator_type": i.indicator_type, + "confidence": i.confidence, + "location": i.location, + "details": i.details, + } + for i in scan_result.stego_indicators + ], + "stego_confidence": scan_result.stego_confidence, + "findings": scan_result.findings, + "created_at": scan_result.created_at.isoformat() if scan_result.created_at else None, + "completed_at": scan_result.completed_at.isoformat() if scan_result.completed_at else None, + } + + return HiddenContentScanResponse(scan=scan_dict, anomaly_created=anomaly_created) + + +@router.post("/hidden-content/quick-scan") +async def quick_scan_hidden_content( + request: Request, + body: HiddenContentQuickScanRequest, +): + """ + Perform quick entropy-only scan on multiple documents. + + Useful for fast screening of large document sets to identify + candidates for full steganography analysis. + + Args: + body: Request with list of doc_ids + + Returns: + List of quick scan results with recommendations + """ + shard = get_shard(request) + + if not shard.hidden_detector: + raise HTTPException( + status_code=503, + detail="Hidden content detector not available" + ) + + if not _db: + raise HTTPException(status_code=503, detail="Database not available") + + results = [] + + for doc_id in body.doc_ids: + try: + # Get document file path + doc_row = await _db.fetch_one( + """SELECT storage_id, metadata FROM arkham_frame.documents WHERE id = :doc_id""", + {"doc_id": doc_id} + ) + + if not doc_row: + results.append({ + "doc_id": doc_id, + "error": "Document not found", + }) + continue + + # Get storage path from storage_id or metadata + storage_id = doc_row.get("storage_id") + doc_metadata = doc_row.get("metadata") or {} + if isinstance(doc_metadata, str): + import json + try: + doc_metadata = json.loads(doc_metadata) + except json.JSONDecodeError: + doc_metadata = {} + + storage_path = doc_metadata.get("storage_path") + if not storage_id and not storage_path: + results.append({ + "doc_id": doc_id, + "error": "Document has no storage path", + }) + continue + + # Read file content + try: + if _storage and storage_id: + file_data = (await _storage.retrieve(storage_id))[0] + elif storage_path: + from pathlib import Path + file_data = Path(storage_path).read_bytes() + else: + raise ValueError("No storage path available") + except Exception as e: + results.append({ + "doc_id": doc_id, + "error": f"Failed to read file: {e}", + }) + continue + + # Quick entropy scan + quick_result = shard.hidden_detector.quick_scan(doc_id, file_data) + results.append(quick_result) + + except Exception as e: + results.append({ + "doc_id": doc_id, + "error": str(e), + }) + + # Summary + requires_full_scan = [r for r in results if r.get("requires_full_scan")] + + return { + "scanned": len(body.doc_ids), + "results": results, + "requires_full_scan_count": len(requires_full_scan), + "requires_full_scan": [r.get("doc_id") for r in requires_full_scan], + } + + +@router.get("/hidden-content/stats", response_model=HiddenContentStatsResponse) +async def get_hidden_content_stats(request: Request): + """ + Get hidden content detection statistics. + + Returns aggregated statistics about hidden content scans: + - Total scans performed + - Scans by type + - Documents with findings + - High entropy files + - Steganography candidates + """ + shard = get_shard(request) + + stats = await shard.get_hidden_content_stats() + return HiddenContentStatsResponse(stats=stats) + + +@router.get("/hidden-content/document/{doc_id}") +async def get_document_hidden_scans( + request: Request, + doc_id: str, +): + """ + Get all hidden content scans for a document. + + Args: + doc_id: Document ID + + Returns: + List of scans for the document + """ + shard = get_shard(request) + + scans = await shard.get_document_hidden_scans(doc_id) + return {"scans": scans, "total": len(scans)} + + +@router.get("/hidden-content/{scan_id}") +async def get_hidden_content_scan( + request: Request, + scan_id: str, +): + """ + Get a specific hidden content scan by ID. + + Args: + scan_id: Scan ID to retrieve + + Returns: + Scan details + """ + shard = get_shard(request) + + scan = await shard.get_hidden_content_scan(scan_id) + if not scan: + raise HTTPException(status_code=404, detail=f"Scan {scan_id} not found") + + return {"scan": scan} diff --git a/packages/arkham-shard-anomalies/arkham_shard_anomalies/hidden_content.py b/packages/arkham-shard-anomalies/arkham_shard_anomalies/hidden_content.py new file mode 100644 index 0000000..12d31c5 --- /dev/null +++ b/packages/arkham-shard-anomalies/arkham_shard_anomalies/hidden_content.py @@ -0,0 +1,531 @@ +"""Hidden content detection module. + +Implements steganography and hidden data detection algorithms: +- Shannon entropy analysis +- LSB (Least Significant Bit) pattern analysis +- Chi-square statistical tests +- File type/magic byte mismatch detection +- Histogram anomaly detection +""" + +import logging +import math +import uuid +from collections import Counter +from datetime import datetime +from typing import Tuple + +import numpy as np +from scipy import stats + +from .models import ( + EntropyRegion, + HiddenContentConfig, + HiddenContentScan, + HiddenContentScanStatus, + HiddenContentScanType, + LSBAnalysisResult, + StegoIndicator, +) + +logger = logging.getLogger(__name__) + + +class HiddenContentDetector: + """ + Detector for hidden content in files. + + Implements multiple detection strategies: + - Shannon entropy analysis for detecting encrypted/compressed/hidden data + - LSB (Least Significant Bit) pattern analysis for steganography + - Chi-square statistical tests for LSB anomalies + - File type/magic byte mismatch detection + - Histogram anomaly detection for images + """ + + def __init__(self, config: HiddenContentConfig | None = None): + self.config = config or HiddenContentConfig() + self._magic = None + + def _get_magic(self): + """Lazy load python-magic with fallback for missing libmagic.""" + if self._magic is None: + try: + import magic + self._magic = magic.Magic(mime=True) + except ImportError: + logger.warning("python-magic not available - file type detection disabled") + self._magic = False # Use False to indicate unavailable + except Exception as e: + # libmagic not installed at OS level + logger.warning(f"libmagic not available: {e} - file type detection disabled") + self._magic = False + return self._magic if self._magic is not False else None + + def calculate_entropy(self, data: bytes) -> float: + """ + Calculate Shannon entropy of byte data. + + Entropy ranges from 0 (no randomness) to 8 (maximum randomness). + High entropy (>7.5) suggests encrypted or compressed data. + + Args: + data: Raw byte data to analyze + + Returns: + Shannon entropy value (0.0 to 8.0) + """ + if not data: + return 0.0 + + byte_counts = Counter(data) + total = len(data) + + entropy = 0.0 + for count in byte_counts.values(): + if count > 0: + probability = count / total + entropy -= probability * math.log2(probability) + + return entropy + + def analyze_entropy_regions( + self, + data: bytes, + chunk_size: int | None = None + ) -> list[EntropyRegion]: + """ + Analyze entropy in chunks to find high-entropy regions. + + High entropy regions may indicate: + - Encrypted data + - Compressed data + - Steganographic payloads + - Random noise (potentially hiding data) + + Args: + data: Raw byte data + chunk_size: Size of each chunk (defaults to config value) + + Returns: + List of entropy regions with anomaly flags + """ + chunk_size = chunk_size or self.config.entropy_chunk_size + regions = [] + threshold = self.config.entropy_threshold_suspicious + + for i in range(0, len(data), chunk_size): + chunk = data[i:i + chunk_size] + if len(chunk) < 64: # Skip tiny trailing chunks + continue + + entropy = self.calculate_entropy(chunk) + + is_anomalous = entropy >= threshold + description = "" + if entropy >= self.config.entropy_threshold_high: + description = "Near-random data (possible encryption/steganography)" + elif entropy >= threshold: + description = "Elevated entropy (suspicious region)" + + regions.append(EntropyRegion( + start_offset=i, + end_offset=min(i + chunk_size, len(data)), + entropy_value=entropy, + is_anomalous=is_anomalous, + description=description, + )) + + return regions + + def analyze_lsb_image(self, image_path: str) -> LSBAnalysisResult | None: + """ + Analyze LSB patterns in an image for steganography detection. + + Uses chi-square analysis on LSB distribution. + Natural images have slight bias in LSB distribution; steganography + tends to create a perfectly uniform 50/50 distribution. + + Args: + image_path: Path to image file + + Returns: + LSB analysis results with suspicion flag, or None if analysis fails + """ + try: + from PIL import Image + except ImportError: + logger.warning("Pillow not available - LSB analysis disabled") + return None + + try: + with Image.open(image_path) as img: + if img.mode not in ('RGB', 'RGBA', 'L'): + img = img.convert('RGB') + + pixels = list(img.getdata()) + sample_size = min(len(pixels), self.config.lsb_sample_size) + + if len(pixels) > sample_size: + indices = np.random.choice(len(pixels), sample_size, replace=False) + pixels = [pixels[i] for i in indices] + + # Extract LSBs from all color channels + lsbs = [] + for pixel in pixels: + if isinstance(pixel, int): + # Grayscale + lsbs.append(pixel & 1) + else: + # RGB or RGBA + for channel in pixel[:3]: + lsbs.append(channel & 1) + + if not lsbs: + return None + + ones = sum(lsbs) + zeros = len(lsbs) - ones + + # Chi-square test against expected 50/50 + expected = len(lsbs) / 2 + chi_square = ((ones - expected) ** 2 + (zeros - expected) ** 2) / expected + p_value = 1 - stats.chi2.cdf(chi_square, df=1) + + bit_ratio = ones / len(lsbs) + # Very close to 50/50 is suspicious for natural images + # Natural images typically have some bias + is_suspicious = ( + p_value > self.config.chi_square_threshold and + 0.48 <= bit_ratio <= 0.52 + ) + + return LSBAnalysisResult( + bit_ratio=bit_ratio, + chi_square_value=chi_square, + chi_square_p_value=p_value, + is_suspicious=is_suspicious, + confidence=1.0 - abs(0.5 - bit_ratio) * 2, + sample_size=len(lsbs), + ) + + except Exception as e: + logger.warning(f"LSB analysis failed for {image_path}: {e}") + return None + + def detect_file_type_mismatch( + self, + file_path: str, + claimed_extension: str + ) -> Tuple[bool, str, str]: + """ + Check if file content matches its extension. + + Mismatches may indicate: + - Renamed files to bypass filters + - Hidden data appended to legitimate files + - Polyglot files (valid as multiple formats) + + Args: + file_path: Path to file + claimed_extension: File extension (e.g., ".jpg") + + Returns: + Tuple of (is_mismatch, expected_mime, actual_mime) + """ + magic = self._get_magic() + if not magic: + return False, "unknown", "unknown" + + try: + actual_mime = magic.from_file(file_path) + except Exception as e: + logger.warning(f"Magic detection failed for {file_path}: {e}") + return False, "unknown", "unknown" + + # Mapping of extensions to expected MIME types + extension_mime_map = { + '.jpg': ['image/jpeg'], + '.jpeg': ['image/jpeg'], + '.png': ['image/png'], + '.gif': ['image/gif'], + '.bmp': ['image/bmp', 'image/x-ms-bmp'], + '.tiff': ['image/tiff'], + '.tif': ['image/tiff'], + '.webp': ['image/webp'], + '.pdf': ['application/pdf'], + '.doc': ['application/msword'], + '.docx': ['application/vnd.openxmlformats-officedocument.wordprocessingml.document'], + '.xls': ['application/vnd.ms-excel'], + '.xlsx': ['application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'], + '.ppt': ['application/vnd.ms-powerpoint'], + '.pptx': ['application/vnd.openxmlformats-officedocument.presentationml.presentation'], + '.txt': ['text/plain'], + '.html': ['text/html'], + '.htm': ['text/html'], + '.xml': ['text/xml', 'application/xml'], + '.json': ['application/json', 'text/json'], + '.zip': ['application/zip'], + '.rar': ['application/x-rar-compressed', 'application/vnd.rar'], + '.7z': ['application/x-7z-compressed'], + '.tar': ['application/x-tar'], + '.gz': ['application/gzip', 'application/x-gzip'], + '.mp3': ['audio/mpeg'], + '.wav': ['audio/wav', 'audio/x-wav'], + '.mp4': ['video/mp4'], + '.avi': ['video/x-msvideo'], + } + + expected_mimes = extension_mime_map.get(claimed_extension.lower(), []) + is_mismatch = actual_mime not in expected_mimes if expected_mimes else False + + return is_mismatch, ', '.join(expected_mimes) if expected_mimes else 'unknown', actual_mime + + def calculate_file_hashes(self, data: bytes) -> dict: + """ + Calculate multiple hash digests for a file. + + Useful for integrity verification and duplicate detection. + + Args: + data: Raw file bytes + + Returns: + Dict with md5, sha256, sha512 hashes + """ + import hashlib + return { + 'md5': hashlib.md5(data).hexdigest(), + 'sha256': hashlib.sha256(data).hexdigest(), + 'sha512': hashlib.sha512(data).hexdigest(), + } + + def analyze_histogram(self, image_path: str) -> dict | None: + """ + Analyze image histogram for anomalies. + + LSB steganography can cause subtle histogram artifacts: + - Pairs of adjacent bins with similar counts (PoV attacks) + - Unusual smoothness in what should be natural distribution + + Args: + image_path: Path to image file + + Returns: + Dict with histogram analysis results, or None if fails + """ + try: + from PIL import Image + except ImportError: + return None + + try: + with Image.open(image_path) as img: + if img.mode != 'RGB': + img = img.convert('RGB') + + # Get histogram for each channel + histogram = img.histogram() + + # Analyze R, G, B channels separately + r_hist = histogram[0:256] + g_hist = histogram[256:512] + b_hist = histogram[512:768] + + # Calculate pair differences for PoV detection + def pair_analysis(hist): + pairs_close = 0 + total_pairs = 0 + for i in range(0, 256, 2): + if hist[i] > 0 or hist[i + 1] > 0: + total_pairs += 1 + if abs(hist[i] - hist[i + 1]) < 10: + pairs_close += 1 + return pairs_close / total_pairs if total_pairs > 0 else 0 + + r_poi = pair_analysis(r_hist) + g_poi = pair_analysis(g_hist) + b_poi = pair_analysis(b_hist) + avg_poi = (r_poi + g_poi + b_poi) / 3 + + # High PoV (pairs of values) ratio is suspicious + is_suspicious = avg_poi > 0.7 + + return { + "r_pair_ratio": r_poi, + "g_pair_ratio": g_poi, + "b_pair_ratio": b_poi, + "average_pair_ratio": avg_poi, + "is_suspicious": is_suspicious, + } + + except Exception as e: + logger.warning(f"Histogram analysis failed: {e}") + return None + + def full_scan( + self, + doc_id: str, + file_path: str, + file_data: bytes, + file_extension: str, + mime_type: str, + ) -> HiddenContentScan: + """ + Perform complete hidden content scan. + + Runs all applicable detection algorithms based on file type: + - Entropy analysis (all files) + - File type mismatch detection (all files) + - LSB analysis (images only) + - Histogram analysis (images only) + + Args: + doc_id: Document identifier + file_path: Path to file on disk + file_data: Raw file bytes + file_extension: File extension + mime_type: MIME type from metadata + + Returns: + Complete scan results + """ + scan = HiddenContentScan( + id=str(uuid.uuid4()), + doc_id=doc_id, + scan_type=HiddenContentScanType.STEGO, + ) + + findings = [] + indicators = [] + + try: + # Check file size limit + file_size_mb = len(file_data) / (1024 * 1024) + if file_size_mb > self.config.max_file_size_mb: + scan.findings = [f"File too large ({file_size_mb:.1f}MB) - skipped"] + scan.scan_status = HiddenContentScanStatus.COMPLETED + scan.completed_at = datetime.utcnow() + return scan + + # 1. Global entropy analysis + if self.config.detect_entropy: + scan.entropy_global = self.calculate_entropy(file_data) + scan.entropy_regions = self.analyze_entropy_regions(file_data) + + high_entropy_regions = [r for r in scan.entropy_regions if r.is_anomalous] + if high_entropy_regions: + findings.append(f"Found {len(high_entropy_regions)} high-entropy regions") + indicators.append(StegoIndicator( + indicator_type="entropy_spike", + confidence=0.7, + location=f"{len(high_entropy_regions)} regions", + details={"region_count": len(high_entropy_regions)}, + )) + + # Check global entropy + if scan.entropy_global and scan.entropy_global >= self.config.entropy_threshold_high: + findings.append(f"Very high global entropy: {scan.entropy_global:.3f}") + indicators.append(StegoIndicator( + indicator_type="high_global_entropy", + confidence=0.8, + location="global", + details={"entropy": scan.entropy_global}, + )) + + # 2. File type mismatch detection + if self.config.detect_magic_mismatch: + is_mismatch, expected, actual = self.detect_file_type_mismatch( + file_path, file_extension + ) + scan.magic_expected = expected + scan.magic_actual = actual + scan.file_mismatch = is_mismatch + + if is_mismatch: + findings.append(f"File type mismatch: expected {expected}, found {actual}") + indicators.append(StegoIndicator( + indicator_type="file_type_mismatch", + confidence=0.9, + location="global", + details={"expected": expected, "actual": actual}, + )) + + # 3. LSB analysis for images + is_image = mime_type and 'image' in mime_type.lower() + if self.config.detect_lsb and is_image: + lsb_result = self.analyze_lsb_image(file_path) + if lsb_result: + scan.lsb_result = lsb_result + + if lsb_result.is_suspicious: + findings.append( + f"Suspicious LSB pattern: {lsb_result.bit_ratio:.3f} ratio, " + f"p-value={lsb_result.chi_square_p_value:.4f}" + ) + indicators.append(StegoIndicator( + indicator_type="lsb_pattern", + confidence=lsb_result.confidence, + location="pixel_lsbs", + details={ + "bit_ratio": lsb_result.bit_ratio, + "chi_square": lsb_result.chi_square_value, + "p_value": lsb_result.chi_square_p_value, + }, + )) + + # 4. Histogram analysis for images + if self.config.detect_histogram and is_image: + hist_result = self.analyze_histogram(file_path) + if hist_result and hist_result.get("is_suspicious"): + findings.append( + f"Suspicious histogram pattern: pair ratio {hist_result['average_pair_ratio']:.3f}" + ) + indicators.append(StegoIndicator( + indicator_type="histogram_anomaly", + confidence=0.6, + location="color_histogram", + details=hist_result, + )) + + scan.findings = findings + scan.stego_indicators = indicators + scan.stego_confidence = max([i.confidence for i in indicators], default=0.0) + scan.scan_status = HiddenContentScanStatus.COMPLETED + scan.completed_at = datetime.utcnow() + + except Exception as e: + logger.error(f"Hidden content scan failed for {doc_id}: {e}", exc_info=True) + scan.scan_status = HiddenContentScanStatus.FAILED + scan.metadata["error"] = str(e) + + return scan + + def quick_scan( + self, + doc_id: str, + file_data: bytes, + ) -> dict: + """ + Perform quick entropy-only scan without file type detection. + + Useful for fast screening of large document sets. + + Args: + doc_id: Document identifier + file_data: Raw file bytes + + Returns: + Dict with quick scan results + """ + global_entropy = self.calculate_entropy(file_data) + regions = self.analyze_entropy_regions(file_data) + high_regions = [r for r in regions if r.is_anomalous] + + return { + "doc_id": doc_id, + "global_entropy": global_entropy, + "is_high_entropy": global_entropy >= self.config.entropy_threshold_high, + "suspicious_regions": len(high_regions), + "requires_full_scan": len(high_regions) > 0 or global_entropy >= self.config.entropy_threshold_suspicious, + } diff --git a/packages/arkham-shard-anomalies/arkham_shard_anomalies/models.py b/packages/arkham-shard-anomalies/arkham_shard_anomalies/models.py index d3a6195..e826a5f 100644 --- a/packages/arkham-shard-anomalies/arkham_shard_anomalies/models.py +++ b/packages/arkham-shard-anomalies/arkham_shard_anomalies/models.py @@ -14,6 +14,11 @@ class AnomalyType(Enum): STRUCTURAL = "structural" # Unusual document structure STATISTICAL = "statistical" # Unusual word frequencies, patterns RED_FLAG = "red_flag" # Sensitive content indicators + # Hidden content types + HIDDEN_CONTENT = "hidden_content" # Steganography or hidden data + FILE_MISMATCH = "file_mismatch" # Extension doesn't match content + HIGH_ENTROPY = "high_entropy" # Unusually high entropy regions + EMBEDDED_PAYLOAD = "embedded_payload" # Embedded files or payloads class AnomalyStatus(Enum): @@ -203,3 +208,130 @@ class AnalystNote: author: str content: str created_at: datetime = field(default_factory=datetime.utcnow) + + +# ============================================================================= +# Hidden Content Detection Models +# ============================================================================= + + +class HiddenContentScanType(Enum): + """Type of hidden content scan.""" + ENTROPY = "entropy" # Shannon entropy analysis + LSB = "lsb" # Least significant bit analysis + MAGIC = "magic" # File type vs extension mismatch + STEGO = "stego" # Full steganography detection + CHI_SQUARE = "chi_square" # Chi-square analysis for LSB + HISTOGRAM = "histogram" # Histogram analysis + + +class HiddenContentScanStatus(Enum): + """Status of hidden content scan.""" + PENDING = "pending" + COMPLETED = "completed" + FAILED = "failed" + + +@dataclass +class EntropyRegion: + """A region of high entropy in a file.""" + start_offset: int + end_offset: int + entropy_value: float + is_anomalous: bool + description: str = "" + + +@dataclass +class LSBAnalysisResult: + """Results from LSB bit pattern analysis.""" + bit_ratio: float # Ratio of 0s to 1s in LSB + chi_square_value: float + chi_square_p_value: float + is_suspicious: bool + confidence: float + sample_size: int + + +@dataclass +class StegoIndicator: + """An indicator of potential steganography.""" + indicator_type: str # "lsb_pattern", "entropy_spike", "chi_square" + confidence: float + location: str # "global", "region:0-1024" + details: dict[str, Any] = field(default_factory=dict) + + +@dataclass +class HiddenContentScan: + """Complete hidden content scan result.""" + id: str + doc_id: str + scan_type: HiddenContentScanType + scan_status: HiddenContentScanStatus = HiddenContentScanStatus.PENDING + + # Entropy analysis + entropy_global: float | None = None + entropy_regions: list[EntropyRegion] = field(default_factory=list) + + # File type analysis + magic_expected: str | None = None + magic_actual: str | None = None + file_mismatch: bool = False + + # LSB analysis + lsb_result: LSBAnalysisResult | None = None + + # Steganography indicators + stego_indicators: list[StegoIndicator] = field(default_factory=list) + stego_confidence: float = 0.0 + + # Findings summary + findings: list[str] = field(default_factory=list) + anomaly_created: bool = False + + # Metadata + created_at: datetime = field(default_factory=datetime.utcnow) + completed_at: datetime | None = None + metadata: dict[str, Any] = field(default_factory=dict) + + +@dataclass +class HiddenContentConfig: + """Configuration for hidden content detection.""" + # Entropy thresholds + entropy_threshold_high: float = 7.5 # Near-random (max 8.0) + entropy_threshold_suspicious: float = 7.0 + entropy_chunk_size: int = 1024 + + # LSB analysis + lsb_sample_size: int = 10000 + chi_square_threshold: float = 0.05 + + # Detection toggles + detect_entropy: bool = True + detect_magic_mismatch: bool = True + detect_lsb: bool = True + detect_chi_square: bool = True + detect_histogram: bool = True + + # File type filters + analyze_images: bool = True + analyze_pdfs: bool = True + analyze_documents: bool = True + + # Processing + max_file_size_mb: int = 100 + timeout_seconds: int = 60 + + +@dataclass +class HiddenContentStats: + """Statistics about hidden content scans.""" + total_scans: int = 0 + scans_by_type: dict[str, int] = field(default_factory=dict) + documents_with_findings: int = 0 + file_type_mismatches: int = 0 + high_entropy_files: int = 0 + stego_candidates: int = 0 + calculated_at: datetime = field(default_factory=datetime.utcnow) diff --git a/packages/arkham-shard-anomalies/arkham_shard_anomalies/shard.py b/packages/arkham-shard-anomalies/arkham_shard_anomalies/shard.py index 9dd7bcd..6d62261 100644 --- a/packages/arkham-shard-anomalies/arkham_shard_anomalies/shard.py +++ b/packages/arkham-shard-anomalies/arkham_shard_anomalies/shard.py @@ -8,8 +8,9 @@ from .api import init_api, router from .detector import AnomalyDetector +from .hidden_content import HiddenContentDetector from .storage import AnomalyStore -from .models import DetectionConfig, Anomaly +from .models import DetectionConfig, Anomaly, HiddenContentConfig, HiddenContentScan logger = logging.getLogger(__name__) @@ -42,13 +43,16 @@ class AnomaliesShard(ArkhamShard): def __init__(self): super().__init__() self.detector: AnomalyDetector | None = None + self.hidden_detector: HiddenContentDetector | None = None self.store: AnomalyStore | None = None self._frame = None self._event_bus = None self._vector_service = None self._db_service = None self._workers = None + self._storage = None self._config = DetectionConfig() + self._hidden_config = HiddenContentConfig() async def initialize(self, frame) -> None: """ @@ -77,15 +81,18 @@ async def initialize(self, frame) -> None: # Get optional services self._event_bus = frame.get_service("events") self._workers = frame.get_service("workers") + self._storage = frame.get_service("storage") # Create database schema await self._create_schema() # Initialize components self.detector = AnomalyDetector(config=self._config) + self.hidden_detector = HiddenContentDetector(config=self._hidden_config) self.store = AnomalyStore(db=self._db_service) logger.info("Anomaly detector initialized") + logger.info("Hidden content detector initialized") logger.info("Anomaly store initialized") # Initialize API @@ -95,6 +102,8 @@ async def initialize(self, frame) -> None: event_bus=self._event_bus, db=self._db_service, vectors=self._vector_service, + hidden_detector=self.hidden_detector, + storage=self._storage, ) # Subscribe to events (correct event names from system) @@ -246,6 +255,88 @@ async def _create_schema(self) -> None: ON arkham_anomaly_patterns(tenant_id) """) + # =========================================== + # Hidden Content Detection Tables + # =========================================== + + # Create schema for new tables (following project conventions) + await self._db_service.execute(""" + CREATE SCHEMA IF NOT EXISTS arkham_anomalies + """) + + # Hidden content scans table + await self._db_service.execute(""" + CREATE TABLE IF NOT EXISTS arkham_anomalies.hidden_content_scans ( + id TEXT PRIMARY KEY, + doc_id TEXT NOT NULL, + scan_type TEXT NOT NULL, + scan_status TEXT DEFAULT 'pending', + findings TEXT DEFAULT '[]', + entropy_score REAL, + entropy_regions TEXT DEFAULT '[]', + magic_expected TEXT, + magic_actual TEXT, + file_signature BYTEA, + lsb_analysis TEXT DEFAULT '{}', + stego_indicators TEXT DEFAULT '[]', + stego_confidence REAL DEFAULT 0.0, + created_at TEXT, + completed_at TEXT, + metadata TEXT DEFAULT '{}', + tenant_id UUID + ) + """) + + # File signatures table for corpus comparison + await self._db_service.execute(""" + CREATE TABLE IF NOT EXISTS arkham_anomalies.file_signatures ( + id TEXT PRIMARY KEY, + doc_id TEXT NOT NULL UNIQUE, + file_hash_md5 TEXT, + file_hash_sha256 TEXT, + file_hash_sha512 TEXT, + magic_type TEXT, + mime_type TEXT, + file_size BIGINT, + entropy_global REAL, + entropy_chunks TEXT DEFAULT '[]', + header_bytes BYTEA, + trailer_bytes BYTEA, + created_at TEXT, + tenant_id UUID + ) + """) + + # Indexes for hidden content tables + await self._db_service.execute(""" + CREATE INDEX IF NOT EXISTS idx_hidden_scans_doc + ON arkham_anomalies.hidden_content_scans(doc_id) + """) + await self._db_service.execute(""" + CREATE INDEX IF NOT EXISTS idx_hidden_scans_type + ON arkham_anomalies.hidden_content_scans(scan_type) + """) + await self._db_service.execute(""" + CREATE INDEX IF NOT EXISTS idx_hidden_scans_status + ON arkham_anomalies.hidden_content_scans(scan_status) + """) + await self._db_service.execute(""" + CREATE INDEX IF NOT EXISTS idx_hidden_scans_tenant + ON arkham_anomalies.hidden_content_scans(tenant_id) + """) + await self._db_service.execute(""" + CREATE INDEX IF NOT EXISTS idx_file_sigs_doc + ON arkham_anomalies.file_signatures(doc_id) + """) + await self._db_service.execute(""" + CREATE INDEX IF NOT EXISTS idx_file_sigs_hash + ON arkham_anomalies.file_signatures(file_hash_sha256) + """) + await self._db_service.execute(""" + CREATE INDEX IF NOT EXISTS idx_file_sigs_tenant + ON arkham_anomalies.file_signatures(tenant_id) + """) + logger.debug("Anomalies schema created/verified") def get_routes(self): @@ -616,18 +707,20 @@ async def _detect_content_anomalies(self, doc_id: str, text: str) -> list[Anomal return [] try: - # Check if vector search is available - if hasattr(self._vector_service, 'search'): - # Search for similar documents - results = await self._vector_service.search( - collection="documents", - query=text[:1000], # Use first 1000 chars as query + # Check if vector text search is available (embeds text then searches) + if hasattr(self._vector_service, 'search_text'): + # Search for similar documents using text query + # search_text handles embedding internally + results = await self._vector_service.search_text( + collection="arkham_documents", # Use correct collection name + text=text[:1000], # Use first 1000 chars as query limit=10, ) # If this document is very different from results, it might be anomalous if results and len(results) > 0: - avg_score = sum(r.get('score', 0) for r in results) / len(results) + # SearchResult is a dataclass with .score attribute + avg_score = sum(r.score if hasattr(r, 'score') else r.get('score', 0) for r in results) / len(results) if avg_score < 0.3: # Low similarity to corpus return [Anomaly( id=str(uuid.uuid4()), @@ -833,3 +926,207 @@ async def get_statistics(self) -> dict: "avg_confidence": stats.avg_confidence, }, } + + # === Hidden Content Detection Methods === + + async def _store_hidden_content_scan(self, scan: HiddenContentScan) -> None: + """ + Store a hidden content scan result in the database. + + Args: + scan: HiddenContentScan result to store + """ + if not self._db_service: + return + + from datetime import datetime + + tenant_id = self.get_tenant_id_or_none() + + # Convert complex objects to JSON strings + entropy_regions = json.dumps([ + { + "start_offset": r.start_offset, + "end_offset": r.end_offset, + "entropy_value": r.entropy_value, + "is_anomalous": r.is_anomalous, + "description": r.description, + } + for r in scan.entropy_regions + ]) + + lsb_analysis = json.dumps( + { + "bit_ratio": scan.lsb_result.bit_ratio, + "chi_square_value": scan.lsb_result.chi_square_value, + "chi_square_p_value": scan.lsb_result.chi_square_p_value, + "is_suspicious": scan.lsb_result.is_suspicious, + "confidence": scan.lsb_result.confidence, + "sample_size": scan.lsb_result.sample_size, + } if scan.lsb_result else {} + ) + + stego_indicators = json.dumps([ + { + "indicator_type": i.indicator_type, + "confidence": i.confidence, + "location": i.location, + "details": i.details, + } + for i in scan.stego_indicators + ]) + + await self._db_service.execute( + """ + INSERT INTO arkham_anomalies.hidden_content_scans ( + id, doc_id, scan_type, scan_status, findings, + entropy_score, entropy_regions, magic_expected, magic_actual, + lsb_analysis, stego_indicators, stego_confidence, + created_at, completed_at, metadata, tenant_id + ) VALUES ( + :id, :doc_id, :scan_type, :scan_status, :findings, + :entropy_score, :entropy_regions, :magic_expected, :magic_actual, + :lsb_analysis, :stego_indicators, :stego_confidence, + :created_at, :completed_at, :metadata, :tenant_id + ) + """, + { + "id": scan.id, + "doc_id": scan.doc_id, + "scan_type": scan.scan_type.value, + "scan_status": scan.scan_status.value, + "findings": json.dumps(scan.findings), + "entropy_score": scan.entropy_global, + "entropy_regions": entropy_regions, + "magic_expected": scan.magic_expected, + "magic_actual": scan.magic_actual, + "lsb_analysis": lsb_analysis, + "stego_indicators": stego_indicators, + "stego_confidence": scan.stego_confidence, + "created_at": scan.created_at.isoformat() if scan.created_at else datetime.utcnow().isoformat(), + "completed_at": scan.completed_at.isoformat() if scan.completed_at else None, + "metadata": json.dumps(scan.metadata), + "tenant_id": str(tenant_id) if tenant_id else None, + } + ) + + async def get_hidden_content_scan(self, scan_id: str) -> Optional[Dict[str, Any]]: + """ + Get a hidden content scan by ID. + + Args: + scan_id: Scan ID + + Returns: + Scan data dict or None + """ + if not self._db_service: + return None + + tenant_id = self.get_tenant_id_or_none() + query = "SELECT * FROM arkham_anomalies.hidden_content_scans WHERE id = :id" + params: Dict[str, Any] = {"id": scan_id} + + if tenant_id: + query += " AND tenant_id = :tenant_id" + params["tenant_id"] = str(tenant_id) + + row = await self._db_service.fetch_one(query, params) + return dict(row) if row else None + + async def get_document_hidden_scans(self, doc_id: str) -> list[Dict[str, Any]]: + """ + Get all hidden content scans for a document. + + Args: + doc_id: Document ID + + Returns: + List of scan data dicts + """ + if not self._db_service: + return [] + + tenant_id = self.get_tenant_id_or_none() + query = """ + SELECT * FROM arkham_anomalies.hidden_content_scans + WHERE doc_id = :doc_id + """ + params: Dict[str, Any] = {"doc_id": doc_id} + + if tenant_id: + query += " AND tenant_id = :tenant_id" + params["tenant_id"] = str(tenant_id) + + query += " ORDER BY created_at DESC" + rows = await self._db_service.fetch_all(query, params) + return [dict(row) for row in rows] if rows else [] + + async def get_hidden_content_stats(self) -> Dict[str, Any]: + """ + Get hidden content detection statistics. + + Returns: + Statistics dictionary + """ + if not self._db_service: + return {} + + tenant_id = self.get_tenant_id_or_none() + tenant_filter = "" + params: Dict[str, Any] = {} + + if tenant_id: + tenant_filter = " WHERE tenant_id = :tenant_id" + params["tenant_id"] = str(tenant_id) + + # Total scans + total_row = await self._db_service.fetch_one( + f"SELECT COUNT(*) as count FROM arkham_anomalies.hidden_content_scans{tenant_filter}", + params + ) + total_scans = total_row.get("count", 0) if total_row else 0 + + # Scans by type + type_rows = await self._db_service.fetch_all( + f"""SELECT scan_type, COUNT(*) as count + FROM arkham_anomalies.hidden_content_scans{tenant_filter} + GROUP BY scan_type""", + params + ) + scans_by_type = {row["scan_type"]: row["count"] for row in type_rows} if type_rows else {} + + # Documents with findings + findings_row = await self._db_service.fetch_one( + f"""SELECT COUNT(DISTINCT doc_id) as count + FROM arkham_anomalies.hidden_content_scans + WHERE findings != '[]'{' AND tenant_id = :tenant_id' if tenant_id else ''}""", + params + ) + docs_with_findings = findings_row.get("count", 0) if findings_row else 0 + + # High entropy files + entropy_row = await self._db_service.fetch_one( + f"""SELECT COUNT(*) as count + FROM arkham_anomalies.hidden_content_scans + WHERE entropy_score >= 7.5{' AND tenant_id = :tenant_id' if tenant_id else ''}""", + params + ) + high_entropy = entropy_row.get("count", 0) if entropy_row else 0 + + # Stego candidates + stego_row = await self._db_service.fetch_one( + f"""SELECT COUNT(*) as count + FROM arkham_anomalies.hidden_content_scans + WHERE stego_confidence >= 0.5{' AND tenant_id = :tenant_id' if tenant_id else ''}""", + params + ) + stego_candidates = stego_row.get("count", 0) if stego_row else 0 + + return { + "total_scans": total_scans, + "scans_by_type": scans_by_type, + "documents_with_findings": docs_with_findings, + "high_entropy_files": high_entropy, + "stego_candidates": stego_candidates, + } diff --git a/packages/arkham-shard-anomalies/pyproject.toml b/packages/arkham-shard-anomalies/pyproject.toml index 83947da..2d1afed 100644 --- a/packages/arkham-shard-anomalies/pyproject.toml +++ b/packages/arkham-shard-anomalies/pyproject.toml @@ -11,6 +11,10 @@ dependencies = [ "pydantic>=2.0.0", "numpy>=1.24.0", "scipy>=1.10.0", + # Hidden content detection dependencies + "python-magic>=0.4.27", # File type detection (requires libmagic system library) + "Pillow>=10.0.0", # Image analysis for LSB steganography detection + "imagehash>=4.3.1", # Perceptual image hashing ] [project.optional-dependencies] diff --git a/packages/arkham-shard-credibility/arkham_shard_credibility/api.py b/packages/arkham-shard-credibility/arkham_shard_credibility/api.py index af5a40b..78abd15 100644 --- a/packages/arkham-shard-credibility/arkham_shard_credibility/api.py +++ b/packages/arkham-shard-credibility/arkham_shard_credibility/api.py @@ -516,7 +516,7 @@ class DeceptionAssessmentCreate(BaseModel): source_name: Optional[str] = Field(None, description="Human-readable source name") linked_assessment_id: Optional[str] = None affects_credibility: bool = True - credibility_weight: float = Field(0.3, ge=0.0, le=1.0) + credibility_weight: float = Field(0.7, ge=0.0, le=1.0, description="Weight in credibility calc (higher = more impact)") class DeceptionChecklistUpdate(BaseModel): diff --git a/packages/arkham-shard-credibility/arkham_shard_credibility/models.py b/packages/arkham-shard-credibility/arkham_shard_credibility/models.py index cf38522..cba643d 100644 --- a/packages/arkham-shard-credibility/arkham_shard_credibility/models.py +++ b/packages/arkham-shard-credibility/arkham_shard_credibility/models.py @@ -352,7 +352,16 @@ def to_dict(self) -> Dict[str, Any]: } def calculate_score(self) -> int: - """Calculate checklist score from indicators.""" + """ + Calculate checklist score from indicators. + + Only assessed indicators (strength != NONE) contribute to the score. + This prevents unassessed indicators from diluting the deception signal. + + Returns 0-100 where: + - 0 = no deception indicators found + - 100 = conclusive deception indicators across all assessed questions + """ if not self.indicators: return 0 @@ -364,11 +373,21 @@ def calculate_score(self) -> int: IndicatorStrength.CONCLUSIVE: 100, } + # Only count indicators that have been assessed (not NONE) + assessed_indicators = [ + ind for ind in self.indicators + if ind.strength != IndicatorStrength.NONE + ] + + if not assessed_indicators: + return 0 + + # Calculate score based only on assessed indicators total = sum( strength_scores.get(ind.strength, 0) * ind.confidence - for ind in self.indicators + for ind in assessed_indicators ) - max_possible = len(self.indicators) * 100 + max_possible = len(assessed_indicators) * 100 return int((total / max_possible) * 100) if max_possible > 0 else 0 @@ -399,7 +418,7 @@ class DeceptionAssessment: # Integration with main credibility linked_assessment_id: Optional[str] = None # Link to CredibilityAssessment affects_credibility: bool = True # Whether to factor into credibility score - credibility_weight: float = 0.3 # Weight in credibility calculation + credibility_weight: float = 0.7 # Weight in credibility calculation (increased for meaningful impact) # Assessment metadata assessed_by: AssessmentMethod = AssessmentMethod.MANUAL diff --git a/packages/arkham-shard-credibility/arkham_shard_credibility/shard.py b/packages/arkham-shard-credibility/arkham_shard_credibility/shard.py index 215c9d8..57d50d6 100644 --- a/packages/arkham-shard-credibility/arkham_shard_credibility/shard.py +++ b/packages/arkham-shard-credibility/arkham_shard_credibility/shard.py @@ -946,6 +946,33 @@ def _parse_llm_assessment(self, response: str) -> Dict[str, Any]: return {"score": 50, "factors": []} + async def _find_existing_credibility_assessment( + self, + source_type: SourceType, + source_id: str, + ) -> Optional[CredibilityAssessment]: + """Find an existing credibility assessment for a source (most recent).""" + if not self._db: + return None + + query = """ + SELECT * FROM arkham_credibility_assessments + WHERE source_type = :source_type AND source_id = :source_id + """ + params = {"source_type": source_type.value, "source_id": source_id} + + tenant_id = self.get_tenant_id_or_none() + if tenant_id: + query += " AND tenant_id = :tenant_id" + params["tenant_id"] = str(tenant_id) + + query += " ORDER BY created_at DESC LIMIT 1" + + row = await self._db.fetch_one(query, params) + if row: + return self._row_to_assessment(row) + return None + # ========================================================================= # DECEPTION DETECTION (MOM/POP/MOSES/EVE) METHODS # ========================================================================= @@ -957,12 +984,19 @@ async def create_deception_assessment( source_name: Optional[str] = None, linked_assessment_id: Optional[str] = None, affects_credibility: bool = True, - credibility_weight: float = 0.3, + credibility_weight: float = 0.7, ) -> DeceptionAssessment: """Create a new deception detection assessment with empty checklists.""" assessment_id = str(uuid4()) now = datetime.utcnow() + # Auto-link to existing credibility assessment if none provided + if not linked_assessment_id and affects_credibility: + existing = await self._find_existing_credibility_assessment(source_type, source_id) + if existing: + linked_assessment_id = existing.id + logger.info(f"Auto-linked deception assessment to existing credibility {existing.id}") + # Create empty checklists with standard indicators mom_checklist = create_empty_checklist(DeceptionChecklistType.MOM) pop_checklist = create_empty_checklist(DeceptionChecklistType.POP) @@ -1361,7 +1395,7 @@ async def _sync_deception_to_credibility(self, deception: DeceptionAssessment) - - 100 = strong deception indicators Credibility score is inverted: high deception = low credibility - The deception score is weighted by credibility_weight (default 0.3). + The deception score is weighted by credibility_weight (default 0.7). """ if not deception.affects_credibility: return @@ -1377,8 +1411,14 @@ async def _sync_deception_to_credibility(self, deception: DeceptionAssessment) - deception_credibility = 100 - deception.overall_score # Create credibility factors from completed checklists + # Use the same weights as the deception assessment calculation + checklist_weights = { + "MOM Analysis": 0.35, # MOM most heavily weighted + "EVE Analysis": 0.25, # Evidence evaluation second + "MOSES Analysis": 0.25, # Manipulability third + "POP Analysis": 0.15, # Past practices least (often unknown) + } factors = [] - checklist_weight = 0.25 # Each checklist gets equal weight for checklist, name in [ (deception.mom_checklist, "MOM Analysis"), @@ -1391,7 +1431,7 @@ async def _sync_deception_to_credibility(self, deception: DeceptionAssessment) - checklist_credibility = 100 - checklist.overall_score factors.append(CredibilityFactor( factor_type=f"deception_{name.lower().replace(' ', '_')}", - weight=checklist_weight, + weight=checklist_weights[name], score=checklist_credibility, notes=f"{name}: {checklist.risk_level} risk ({checklist.overall_score}% deception indicators)", )) @@ -1406,6 +1446,25 @@ async def _sync_deception_to_credibility(self, deception: DeceptionAssessment) - else: final_score = deception_credibility + # SEVERITY OVERRIDE: High deception scores force low credibility + # This ensures that conclusive/strong deception evidence dramatically impacts credibility + # regardless of how other factors might average out + if deception.overall_score >= 80: # Critical risk + # Conclusive deception: cap credibility at 15 (Unreliable) + final_score = min(final_score, 15) + logger.info(f"Deception severity override: critical risk ({deception.overall_score}%) " + f"capped credibility at {final_score}") + elif deception.overall_score >= 60: # High risk + # Strong deception: cap credibility at 30 (Low) + final_score = min(final_score, 30) + logger.info(f"Deception severity override: high risk ({deception.overall_score}%) " + f"capped credibility at {final_score}") + elif deception.overall_score >= 40: # Moderate risk + # Moderate deception: cap credibility at 50 (Medium) + final_score = min(final_score, 50) + logger.info(f"Deception severity override: moderate risk ({deception.overall_score}%) " + f"capped credibility at {final_score}") + # Confidence based on how many checklists were completed confidence = deception.confidence if deception.confidence > 0 else 0.5 @@ -1573,7 +1632,7 @@ def parse_checklist(data_str: str, checklist_type: DeceptionChecklistType) -> Op confidence=row.get("confidence", 0.0), linked_assessment_id=row.get("linked_assessment_id"), affects_credibility=bool(row.get("affects_credibility", 1)), - credibility_weight=row.get("credibility_weight", 0.3), + credibility_weight=row.get("credibility_weight", 0.7), assessed_by=AssessmentMethod(row.get("assessed_by", "manual")), assessor_id=row.get("assessor_id"), summary=row.get("summary"), diff --git a/packages/arkham-shard-documents/arkham_shard_documents/api.py b/packages/arkham-shard-documents/arkham_shard_documents/api.py index be06235..3d9d832 100644 --- a/packages/arkham-shard-documents/arkham_shard_documents/api.py +++ b/packages/arkham-shard-documents/arkham_shard_documents/api.py @@ -662,3 +662,261 @@ async def ai_junior_analyst(request: Request, body: AIJuniorAnalystRequest): "X-Accel-Buffering": "no", }, ) + + +# --- Deduplication Endpoints --- + + +class DuplicateMatchResponse(BaseModel): + """A potential duplicate match.""" + document_id: str + title: str + similarity_score: float + hamming_distance: int + match_type: str + + +class DuplicateGroupResponse(BaseModel): + """A group of duplicate documents.""" + group_id: str + primary_document_id: str + duplicate_ids: List[str] + similarity_threshold: float + detection_method: str + + +class MergeRequest(BaseModel): + """Request to merge duplicate documents.""" + primary_id: str + duplicate_ids: List[str] + strategy: str = "keep_primary" + preserve_references: bool = True + cleanup_action: str = "soft_delete" + + +class MergeResponse(BaseModel): + """Response from merge operation.""" + primary_id: str + merged_count: int + references_updated: int + documents_cleaned: int + cleanup_action: str + + +class DeduplicationStats(BaseModel): + """Deduplication statistics.""" + total_documents: int + documents_with_hash: int + unique_content_hashes: int + potential_duplicates: int + + +@router.post("/{document_id}/compute-hash") +async def compute_document_hash(document_id: str, request: Request): + """ + Compute and store content hashes for a document. + + Computes MD5, SHA256, and SimHash for deduplication. + """ + shard = get_shard(request) + + if not shard.deduplication: + raise HTTPException(status_code=503, detail="Deduplication service not available") + + # Get document content + content = await shard.get_document_content(document_id) + if not content: + raise HTTPException(status_code=404, detail=f"Document content not found: {document_id}") + + # Compute hashes + result = await shard.deduplication.compute_hash( + document_id=document_id, + text=content["content"], + store=True, + ) + + return { + "document_id": document_id, + "content_md5": result["content_md5"], + "content_sha256": result["content_sha256"], + "simhash": result["simhash"], + "text_length": result["text_length"], + } + + +@router.get("/{document_id}/duplicates/exact", response_model=List[DuplicateMatchResponse]) +async def find_exact_duplicates( + document_id: str, + request: Request, + project_id: Optional[str] = Query(None, description="Limit to project"), +): + """ + Find exact duplicates of a document. + + Uses content SHA256 hash for exact matching. + """ + shard = get_shard(request) + + if not shard.deduplication: + raise HTTPException(status_code=503, detail="Deduplication service not available") + + matches = await shard.deduplication.find_exact_duplicates( + document_id=document_id, + project_id=project_id, + ) + + return [ + DuplicateMatchResponse( + document_id=m.document_id, + title=m.title, + similarity_score=m.similarity_score, + hamming_distance=m.hamming_distance, + match_type=m.match_type, + ) + for m in matches + ] + + +@router.get("/{document_id}/duplicates/similar", response_model=List[DuplicateMatchResponse]) +async def find_similar_documents( + document_id: str, + request: Request, + threshold: float = Query(0.85, ge=0.0, le=1.0, description="Similarity threshold"), + project_id: Optional[str] = Query(None, description="Limit to project"), + limit: int = Query(20, ge=1, le=100, description="Maximum results"), +): + """ + Find similar documents using SimHash. + + Returns documents with similarity above threshold. + """ + shard = get_shard(request) + + if not shard.deduplication: + raise HTTPException(status_code=503, detail="Deduplication service not available") + + matches = await shard.deduplication.find_similar_documents( + document_id=document_id, + threshold=threshold, + project_id=project_id, + limit=limit, + ) + + return [ + DuplicateMatchResponse( + document_id=m.document_id, + title=m.title, + similarity_score=m.similarity_score, + hamming_distance=m.hamming_distance, + match_type=m.match_type, + ) + for m in matches + ] + + +@router.post("/deduplication/scan", response_model=List[DuplicateGroupResponse]) +async def scan_project_duplicates( + request: Request, + project_id: str = Query(..., description="Project to scan"), + threshold: float = Query(0.85, ge=0.0, le=1.0, description="Similarity threshold"), +): + """ + Scan entire project for duplicate groups. + + Returns groups of documents that appear to be duplicates. + """ + shard = get_shard(request) + + if not shard.deduplication: + raise HTTPException(status_code=503, detail="Deduplication service not available") + + groups = await shard.deduplication.scan_project_duplicates( + project_id=project_id, + threshold=threshold, + ) + + return [ + DuplicateGroupResponse( + group_id=g.group_id, + primary_document_id=g.primary_document_id, + duplicate_ids=g.duplicate_ids, + similarity_threshold=g.similarity_threshold, + detection_method=g.detection_method, + ) + for g in groups + ] + + +@router.post("/deduplication/merge", response_model=MergeResponse) +async def merge_duplicates(body: MergeRequest, request: Request): + """ + Merge duplicate documents into a primary document. + + Cleanup actions: + - soft_delete: Mark duplicates as merged (recommended) + - archive: Move to archive status + - hard_delete: Permanently delete (irreversible) + - keep: Only update references, keep duplicates + """ + shard = get_shard(request) + + if not shard.deduplication: + raise HTTPException(status_code=503, detail="Deduplication service not available") + + # Validate primary document exists + primary_doc = await shard.get_document(body.primary_id) + if not primary_doc: + raise HTTPException(status_code=404, detail=f"Primary document not found: {body.primary_id}") + + result = await shard.deduplication.merge_documents( + primary_id=body.primary_id, + duplicate_ids=body.duplicate_ids, + strategy=body.strategy, + preserve_references=body.preserve_references, + cleanup_action=body.cleanup_action, + ) + + # Emit event + if shard._events: + await shard._events.emit( + "documents.duplicates.merged", + { + "primary_id": result.primary_id, + "merged_count": result.merged_count, + "cleanup_action": result.cleanup_action, + }, + source="documents-shard", + ) + + return MergeResponse( + primary_id=result.primary_id, + merged_count=result.merged_count, + references_updated=result.references_updated, + documents_cleaned=result.documents_cleaned, + cleanup_action=result.cleanup_action, + ) + + +@router.get("/deduplication/stats", response_model=DeduplicationStats) +async def get_deduplication_stats( + request: Request, + project_id: Optional[str] = Query(None, description="Filter by project"), +): + """ + Get deduplication statistics. + + Returns counts of documents, unique hashes, and potential duplicates. + """ + shard = get_shard(request) + + if not shard.deduplication: + raise HTTPException(status_code=503, detail="Deduplication service not available") + + stats = await shard.deduplication.get_deduplication_stats(project_id=project_id) + + return DeduplicationStats( + total_documents=stats.get("total_documents", 0), + documents_with_hash=stats.get("documents_with_hash", 0), + unique_content_hashes=stats.get("unique_content_hashes", 0), + potential_duplicates=stats.get("potential_duplicates", 0), + ) diff --git a/packages/arkham-shard-documents/arkham_shard_documents/models.py b/packages/arkham-shard-documents/arkham_shard_documents/models.py index b385974..8bcc7af 100644 --- a/packages/arkham-shard-documents/arkham_shard_documents/models.py +++ b/packages/arkham-shard-documents/arkham_shard_documents/models.py @@ -13,6 +13,7 @@ class DocumentStatus(Enum): PROCESSED = "processed" FAILED = "failed" ARCHIVED = "archived" + MERGED = "merged" class ViewMode(Enum): diff --git a/packages/arkham-shard-documents/arkham_shard_documents/services/__init__.py b/packages/arkham-shard-documents/arkham_shard_documents/services/__init__.py new file mode 100644 index 0000000..c08c974 --- /dev/null +++ b/packages/arkham-shard-documents/arkham_shard_documents/services/__init__.py @@ -0,0 +1,5 @@ +"""Documents shard services.""" + +from .deduplication import DeduplicationService + +__all__ = ["DeduplicationService"] diff --git a/packages/arkham-shard-documents/arkham_shard_documents/services/deduplication.py b/packages/arkham-shard-documents/arkham_shard_documents/services/deduplication.py new file mode 100644 index 0000000..3717043 --- /dev/null +++ b/packages/arkham-shard-documents/arkham_shard_documents/services/deduplication.py @@ -0,0 +1,701 @@ +"""Document deduplication service using SimHash for similarity detection.""" + +import hashlib +import json +import logging +import re +import uuid +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any + +logger = logging.getLogger(__name__) + + +@dataclass +class DuplicateMatch: + """A potential duplicate match.""" + document_id: str + title: str + similarity_score: float + hamming_distance: int + match_type: str # exact, near, content_similar + shared_entities: int = 0 + metadata: dict[str, Any] = field(default_factory=dict) + + +@dataclass +class DuplicateGroup: + """A group of duplicate documents.""" + group_id: str + primary_document_id: str + duplicate_ids: list[str] + similarity_threshold: float + detection_method: str + created_at: datetime = field(default_factory=datetime.utcnow) + + +@dataclass +class MergeResult: + """Result of merging duplicate documents.""" + primary_id: str + merged_count: int + references_updated: int + documents_cleaned: int + cleanup_action: str # soft_delete, archive, hard_delete, keep + + +class SimHash: + """ + SimHash implementation for text similarity detection. + + SimHash produces a 64-bit hash where similar documents have similar hashes. + Hamming distance between hashes indicates similarity. + """ + + def __init__(self, hash_bits: int = 64): + """ + Initialize SimHash. + + Args: + hash_bits: Number of bits in hash (default: 64) + """ + self.hash_bits = hash_bits + + def _tokenize(self, text: str) -> list[str]: + """ + Tokenize text into shingles (n-grams). + + Args: + text: Input text + + Returns: + List of token strings + """ + # Normalize text + text = text.lower() + text = re.sub(r'[^\w\s]', '', text) + + # Generate word shingles (3-grams) + words = text.split() + shingles = [] + + # Unigrams + shingles.extend(words) + + # Bigrams + for i in range(len(words) - 1): + shingles.append(f"{words[i]} {words[i+1]}") + + # Trigrams + for i in range(len(words) - 2): + shingles.append(f"{words[i]} {words[i+1]} {words[i+2]}") + + return shingles + + def _hash_token(self, token: str) -> int: + """ + Hash a single token to a 64-bit integer. + + Args: + token: Token string + + Returns: + 64-bit hash integer + """ + # Use MD5 and take first 8 bytes (64 bits) + h = hashlib.md5(token.encode('utf-8')).digest() + return int.from_bytes(h[:8], byteorder='big') + + def compute(self, text: str) -> int: + """ + Compute SimHash for a text. + + Args: + text: Input text + + Returns: + 64-bit SimHash integer + """ + if not text or not text.strip(): + return 0 + + tokens = self._tokenize(text) + if not tokens: + return 0 + + # Initialize bit vector + v = [0] * self.hash_bits + + # Accumulate weighted hash bits + for token in tokens: + token_hash = self._hash_token(token) + + # Add/subtract based on each bit + for i in range(self.hash_bits): + bit = (token_hash >> (self.hash_bits - 1 - i)) & 1 + if bit == 1: + v[i] += 1 + else: + v[i] -= 1 + + # Generate final hash from accumulated values + simhash = 0 + for i in range(self.hash_bits): + if v[i] > 0: + simhash |= (1 << (self.hash_bits - 1 - i)) + + return simhash + + @staticmethod + def hamming_distance(hash1: int, hash2: int) -> int: + """ + Calculate Hamming distance between two hashes. + + Args: + hash1: First hash + hash2: Second hash + + Returns: + Number of differing bits (0-64) + """ + xor = hash1 ^ hash2 + distance = 0 + while xor: + distance += xor & 1 + xor >>= 1 + return distance + + @staticmethod + def similarity_score(hash1: int, hash2: int, hash_bits: int = 64) -> float: + """ + Calculate similarity score from Hamming distance. + + Args: + hash1: First hash + hash2: Second hash + hash_bits: Number of bits in hash + + Returns: + Similarity score (0.0-1.0) + """ + distance = SimHash.hamming_distance(hash1, hash2) + return 1.0 - (distance / hash_bits) + + +class DeduplicationService: + """ + Document deduplication service. + + Provides: + - SimHash-based similarity detection + - Exact match detection (MD5/SHA256) + - Duplicate grouping + - Merge operations with cleanup options + """ + + def __init__(self, database_service, config: dict | None = None): + """ + Initialize deduplication service. + + Args: + database_service: Frame database service + config: Optional configuration dict + """ + self._db = database_service + self._config = config or {} + self._simhash = SimHash(hash_bits=64) + + # Configuration + self._similarity_threshold = self._config.get("similarity_threshold", 0.85) + self._hamming_threshold = self._config.get("hamming_threshold", 10) + + async def compute_hash( + self, + document_id: str, + text: str, + store: bool = True, + ) -> dict[str, Any]: + """ + Compute and store content hashes for a document. + + Args: + document_id: Document ID + text: Document text content + store: Whether to store hashes in database + + Returns: + Dict with hash values + """ + # Compute hashes + content_md5 = hashlib.md5(text.encode('utf-8')).hexdigest() + content_sha256 = hashlib.sha256(text.encode('utf-8')).hexdigest() + simhash = self._simhash.compute(text) + + result = { + "document_id": document_id, + "content_md5": content_md5, + "content_sha256": content_sha256, + "simhash": simhash, + "text_length": len(text), + } + + if store and self._db: + try: + # Upsert into content_hashes table + await self._db.execute( + """INSERT INTO arkham_documents.content_hashes + (id, document_id, content_md5, content_sha256, simhash, text_length, created_at) + VALUES (:id, :doc_id, :md5, :sha256, :simhash, :text_len, CURRENT_TIMESTAMP) + ON CONFLICT (document_id) DO UPDATE SET + content_md5 = EXCLUDED.content_md5, + content_sha256 = EXCLUDED.content_sha256, + simhash = EXCLUDED.simhash, + text_length = EXCLUDED.text_length, + created_at = CURRENT_TIMESTAMP""", + { + "id": str(uuid.uuid4()), + "doc_id": document_id, + "md5": content_md5, + "sha256": content_sha256, + "simhash": simhash, + "text_len": len(text), + } + ) + except Exception as e: + logger.warning(f"Failed to store content hashes: {e}") + + return result + + async def find_exact_duplicates( + self, + document_id: str, + project_id: str | None = None, + ) -> list[DuplicateMatch]: + """ + Find exact duplicates using content hash. + + Args: + document_id: Source document ID + project_id: Optional project scope + + Returns: + List of exact duplicate matches + """ + if not self._db: + return [] + + # Get source document's hash + source_row = await self._db.fetch_one( + "SELECT * FROM arkham_documents.content_hashes WHERE document_id = :doc_id", + {"doc_id": document_id} + ) + + if not source_row: + return [] + + source_hash = source_row["content_sha256"] + + # Find documents with matching hash + query = """ + SELECT ch.document_id, ch.content_sha256, d.filename as title + FROM arkham_documents.content_hashes ch + JOIN arkham_frame.documents d ON ch.document_id = d.id + WHERE ch.content_sha256 = :hash + AND ch.document_id != :source_id + """ + params: dict[str, Any] = {"hash": source_hash, "source_id": document_id} + + if project_id: + query += " AND d.project_id = :project_id" + params["project_id"] = project_id + + rows = await self._db.fetch_all(query, params) + + return [ + DuplicateMatch( + document_id=row["document_id"], + title=row["title"] or "", + similarity_score=1.0, + hamming_distance=0, + match_type="exact", + ) + for row in rows + ] + + async def find_similar_documents( + self, + document_id: str, + threshold: float | None = None, + project_id: str | None = None, + limit: int = 20, + ) -> list[DuplicateMatch]: + """ + Find similar documents using SimHash. + + Args: + document_id: Source document ID + threshold: Similarity threshold (0.0-1.0) + project_id: Optional project scope + limit: Maximum results + + Returns: + List of similar document matches + """ + if not self._db: + return [] + + threshold = threshold or self._similarity_threshold + + # Get source document's simhash + source_row = await self._db.fetch_one( + "SELECT * FROM arkham_documents.content_hashes WHERE document_id = :doc_id", + {"doc_id": document_id} + ) + + if not source_row or source_row["simhash"] is None: + return [] + + source_simhash = source_row["simhash"] + + # Get all other documents' simhashes (in batches for large corpora) + query = """ + SELECT ch.document_id, ch.simhash, d.filename as title + FROM arkham_documents.content_hashes ch + JOIN arkham_frame.documents d ON ch.document_id = d.id + WHERE ch.document_id != :source_id + AND ch.simhash IS NOT NULL + """ + params: dict[str, Any] = {"source_id": document_id} + + if project_id: + query += " AND d.project_id = :project_id" + params["project_id"] = project_id + + rows = await self._db.fetch_all(query, params) + + # Calculate similarity for each + similar = [] + for row in rows: + other_simhash = row["simhash"] + if other_simhash is None: + continue + + similarity = SimHash.similarity_score(source_simhash, other_simhash) + hamming = SimHash.hamming_distance(source_simhash, other_simhash) + + if similarity >= threshold: + match_type = "exact" if hamming == 0 else "near" if hamming <= 5 else "content_similar" + + similar.append(DuplicateMatch( + document_id=row["document_id"], + title=row["title"] or "", + similarity_score=round(similarity, 4), + hamming_distance=hamming, + match_type=match_type, + )) + + # Sort by similarity descending + similar.sort(key=lambda x: x.similarity_score, reverse=True) + + return similar[:limit] + + async def scan_project_duplicates( + self, + project_id: str, + threshold: float | None = None, + ) -> list[DuplicateGroup]: + """ + Scan entire project for duplicate groups. + + Uses efficient pairwise comparison with early termination. + + Args: + project_id: Project to scan + threshold: Similarity threshold + + Returns: + List of duplicate groups found + """ + if not self._db: + return [] + + threshold = threshold or self._similarity_threshold + + # Get all documents with simhashes in project + rows = await self._db.fetch_all( + """SELECT ch.document_id, ch.simhash, d.filename as title + FROM arkham_documents.content_hashes ch + JOIN arkham_frame.documents d ON ch.document_id = d.id + WHERE d.project_id = :project_id + AND ch.simhash IS NOT NULL + ORDER BY d.created_at""", + {"project_id": project_id} + ) + + if len(rows) < 2: + return [] + + # Build duplicate groups using union-find + groups: dict[str, set[str]] = {} # primary_id -> set of duplicate ids + assigned: dict[str, str] = {} # doc_id -> group primary_id + + for i, row1 in enumerate(rows): + doc1_id = row1["document_id"] + simhash1 = row1["simhash"] + + for j, row2 in enumerate(rows): + if j <= i: + continue + + doc2_id = row2["document_id"] + simhash2 = row2["simhash"] + + similarity = SimHash.similarity_score(simhash1, simhash2) + + if similarity >= threshold: + # Determine which group these belong to + group1 = assigned.get(doc1_id) + group2 = assigned.get(doc2_id) + + if group1 is None and group2 is None: + # Create new group with doc1 as primary + groups[doc1_id] = {doc1_id, doc2_id} + assigned[doc1_id] = doc1_id + assigned[doc2_id] = doc1_id + elif group1 is not None and group2 is None: + # Add doc2 to group1 + groups[group1].add(doc2_id) + assigned[doc2_id] = group1 + elif group1 is None and group2 is not None: + # Add doc1 to group2 + groups[group2].add(doc1_id) + assigned[doc1_id] = group2 + elif group1 != group2: + # Merge groups (keep smaller primary_id as primary) + primary = min(group1, group2) + secondary = max(group1, group2) + groups[primary].update(groups[secondary]) + for doc_id in groups[secondary]: + assigned[doc_id] = primary + del groups[secondary] + + # Convert to DuplicateGroup objects + result = [] + for primary_id, members in groups.items(): + if len(members) > 1: + duplicate_ids = [m for m in members if m != primary_id] + result.append(DuplicateGroup( + group_id=str(uuid.uuid4())[:8], + primary_document_id=primary_id, + duplicate_ids=duplicate_ids, + similarity_threshold=threshold, + detection_method="simhash", + )) + + return result + + async def merge_documents( + self, + primary_id: str, + duplicate_ids: list[str], + strategy: str = "keep_primary", + preserve_references: bool = True, + cleanup_action: str = "soft_delete", + ) -> MergeResult: + """ + Merge duplicate documents into a primary document. + + Args: + primary_id: Document to keep + duplicate_ids: Documents to merge into primary + strategy: Merge strategy (keep_primary, merge_metadata, keep_longest) + preserve_references: Update references from duplicates to primary + cleanup_action: What to do with duplicate documents after merge: + - "soft_delete": Mark as deleted, set merged_into_id (default, audit-safe) + - "archive": Move to archive status, keep all data + - "hard_delete": Permanently delete (data loss, not recommended) + - "keep": Leave duplicates in place (just update references) + + Returns: + MergeResult with details + + Cleanup Actions Explained: + ------------------------- + 1. soft_delete (DEFAULT - Recommended for audit trails): + - Sets status = 'merged' on duplicate documents + - Adds merged_into_id = primary_id for traceability + - Adds merged_at timestamp + - Document remains in database but excluded from normal queries + - Can be fully restored if merge was incorrect + + 2. archive: + - Sets status = 'archived' on duplicate documents + - Document remains fully accessible via archive views + - Best for legal/compliance scenarios requiring data retention + + 3. hard_delete: + - Permanently removes duplicate documents from database + - Also removes associated chunks, embeddings, hashes + - IRREVERSIBLE - only use when storage is critical + - Merge history still preserved for audit + + 4. keep: + - Only updates references, leaves duplicates unchanged + - Useful for testing or when duplicates serve different purposes + """ + references_updated = 0 + documents_cleaned = 0 + + if preserve_references: + # Update entity mentions + for dup_id in duplicate_ids: + try: + result = await self._db.execute( + """UPDATE arkham_entity_mentions + SET document_id = :primary_id + WHERE document_id = :dup_id""", + {"primary_id": primary_id, "dup_id": dup_id} + ) + references_updated += getattr(result, 'rowcount', 0) + except Exception as e: + logger.warning(f"Failed to update entity mentions: {e}") + + # Update claims + try: + result = await self._db.execute( + """UPDATE arkham_claims + SET document_id = :primary_id + WHERE document_id = :dup_id""", + {"primary_id": primary_id, "dup_id": dup_id} + ) + references_updated += getattr(result, 'rowcount', 0) + except Exception as e: + logger.warning(f"Failed to update claims: {e}") + + # ===================================================== + # CLEANUP: Handle duplicate documents based on cleanup_action + # ===================================================== + for dup_id in duplicate_ids: + try: + if cleanup_action == "soft_delete": + # Recommended: Soft delete with merge pointer for audit trail + await self._db.execute( + """UPDATE arkham_frame.documents + SET status = 'merged', + metadata = jsonb_set( + COALESCE(metadata::jsonb, '{}'), + '{merged_into_id}', + :primary_json + ), + updated_at = CURRENT_TIMESTAMP + WHERE id = :dup_id""", + {"primary_json": json.dumps(primary_id), "dup_id": dup_id} + ) + documents_cleaned += 1 + + elif cleanup_action == "archive": + # Archive: Keep fully accessible but mark as archived + await self._db.execute( + """UPDATE arkham_frame.documents + SET status = 'archived', + metadata = jsonb_set( + COALESCE(metadata::jsonb, '{}'), + '{archived_reason}', + :reason_json + ), + updated_at = CURRENT_TIMESTAMP + WHERE id = :dup_id""", + {"reason_json": json.dumps(f"duplicate_of_{primary_id}"), "dup_id": dup_id} + ) + documents_cleaned += 1 + + elif cleanup_action == "hard_delete": + # DANGER: Permanent deletion - use with caution + # Delete in order: chunks -> embeddings -> hashes -> document + await self._db.execute( + "DELETE FROM arkham_frame.chunks WHERE document_id = :dup_id", + {"dup_id": dup_id} + ) + await self._db.execute( + "DELETE FROM arkham_documents.content_hashes WHERE document_id = :dup_id", + {"dup_id": dup_id} + ) + await self._db.execute( + "DELETE FROM arkham_frame.documents WHERE id = :dup_id", + {"dup_id": dup_id} + ) + documents_cleaned += 1 + + elif cleanup_action == "keep": + # Do nothing - just updated references above + pass + + except Exception as e: + logger.warning(f"Failed to cleanup duplicate {dup_id}: {e}") + + # Record merge in history (always, regardless of cleanup action) + await self._db.execute( + """INSERT INTO arkham_documents.merge_history + (id, primary_document_id, merged_document_ids, strategy, cleanup_action, + references_updated, documents_cleaned, merged_at) + VALUES (:id, :primary_id, :dup_ids, :strategy, :cleanup, :refs, :cleaned, CURRENT_TIMESTAMP)""", + { + "id": str(uuid.uuid4()), + "primary_id": primary_id, + "dup_ids": json.dumps(duplicate_ids), + "strategy": strategy, + "cleanup": cleanup_action, + "refs": references_updated, + "cleaned": documents_cleaned, + } + ) + + return MergeResult( + primary_id=primary_id, + merged_count=len(duplicate_ids), + references_updated=references_updated, + documents_cleaned=documents_cleaned, + cleanup_action=cleanup_action, + ) + + async def get_deduplication_stats(self, project_id: str | None = None) -> dict[str, Any]: + """ + Get deduplication statistics. + + Args: + project_id: Optional project scope + + Returns: + Dict with statistics + """ + if not self._db: + return {} + + query = """ + SELECT COUNT(*) as total_documents, + COUNT(ch.id) as documents_with_hash, + COUNT(DISTINCT ch.content_sha256) as unique_content_hashes + FROM arkham_frame.documents d + LEFT JOIN arkham_documents.content_hashes ch ON d.id = ch.document_id + """ + params: dict[str, Any] = {} + + if project_id: + query += " WHERE d.project_id = :project_id" + params["project_id"] = project_id + + row = await self._db.fetch_one(query, params) + + stats = { + "total_documents": row["total_documents"] if row else 0, + "documents_with_hash": row["documents_with_hash"] if row else 0, + "unique_content_hashes": row["unique_content_hashes"] if row else 0, + "potential_duplicates": 0, + } + + # Calculate potential duplicates + if row and row["documents_with_hash"] > row["unique_content_hashes"]: + stats["potential_duplicates"] = row["documents_with_hash"] - row["unique_content_hashes"] + + return stats diff --git a/packages/arkham-shard-documents/arkham_shard_documents/shard.py b/packages/arkham-shard-documents/arkham_shard_documents/shard.py index 68f0479..b43b3dc 100644 --- a/packages/arkham-shard-documents/arkham_shard_documents/shard.py +++ b/packages/arkham-shard-documents/arkham_shard_documents/shard.py @@ -9,6 +9,7 @@ from .api import router from .models import DocumentRecord, DocumentStatus +from .services import DeduplicationService logger = logging.getLogger(__name__) @@ -49,6 +50,7 @@ def __init__(self): self._events = None self._storage = None self._document_service = None + self._deduplication_service = None async def initialize(self, frame) -> None: """ @@ -78,6 +80,10 @@ async def initialize(self, frame) -> None: if not self._document_service: logger.warning("Document service not available - using database directly") + # Initialize deduplication service + self._deduplication_service = DeduplicationService(self._db) + logger.info("Deduplication service initialized") + # Create database schema await self._create_schema() @@ -108,6 +114,7 @@ async def shutdown(self) -> None: self._events = None self._storage = None self._document_service = None + self._deduplication_service = None logger.info("Documents Shard shutdown complete") @@ -115,6 +122,11 @@ def get_routes(self): """Return FastAPI router for this shard.""" return router + @property + def deduplication(self) -> DeduplicationService | None: + """Get deduplication service.""" + return self._deduplication_service + async def _create_schema(self) -> None: """ Create database schema for documents shard. @@ -262,6 +274,63 @@ async def _create_schema(self) -> None: ON arkham_document_views(tenant_id) """) + # =========================================== + # Deduplication Tables (arkham_documents schema) + # =========================================== + await self._db.execute("CREATE SCHEMA IF NOT EXISTS arkham_documents") + + # Content hashes for deduplication + await self._db.execute(""" + CREATE TABLE IF NOT EXISTS arkham_documents.content_hashes ( + id TEXT PRIMARY KEY, + document_id TEXT NOT NULL UNIQUE, + content_md5 TEXT, + content_sha256 TEXT, + simhash BIGINT, + text_length INTEGER, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + tenant_id UUID + ) + """) + + # Merge history for audit trail + await self._db.execute(""" + CREATE TABLE IF NOT EXISTS arkham_documents.merge_history ( + id TEXT PRIMARY KEY, + primary_document_id TEXT NOT NULL, + merged_document_ids JSONB NOT NULL, + strategy TEXT, + cleanup_action TEXT, + references_updated INTEGER DEFAULT 0, + documents_cleaned INTEGER DEFAULT 0, + merged_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + merged_by TEXT, + tenant_id UUID + ) + """) + + # Indexes for deduplication + await self._db.execute(""" + CREATE INDEX IF NOT EXISTS idx_content_hashes_doc + ON arkham_documents.content_hashes(document_id) + """) + await self._db.execute(""" + CREATE INDEX IF NOT EXISTS idx_content_hashes_md5 + ON arkham_documents.content_hashes(content_md5) + """) + await self._db.execute(""" + CREATE INDEX IF NOT EXISTS idx_content_hashes_sha256 + ON arkham_documents.content_hashes(content_sha256) + """) + await self._db.execute(""" + CREATE INDEX IF NOT EXISTS idx_content_hashes_simhash + ON arkham_documents.content_hashes(simhash) + """) + await self._db.execute(""" + CREATE INDEX IF NOT EXISTS idx_merge_history_primary + ON arkham_documents.merge_history(primary_document_id) + """) + logger.info("Documents shard database schema created") # --- Helper Methods --- @@ -324,8 +393,8 @@ def _frame_row_to_document(self, row: Dict[str, Any], entity_count: int = 0) -> id=row["id"], title=row.get("filename", ""), # Frame uses filename, no separate title filename=row.get("filename", ""), - file_type=row.get("mime_type", ""), - file_size=row.get("file_size", 0), + file_type=row.get("mime_type") or "", # Handle None values + file_size=row.get("file_size") or 0, status=DocumentStatus(row.get("status", "pending")), page_count=row.get("page_count", 0), chunk_count=row.get("chunk_count", 0), diff --git a/packages/arkham-shard-embed/arkham_shard_embed/api.py b/packages/arkham-shard-embed/arkham_shard_embed/api.py index 15b1a14..6a63ea0 100644 --- a/packages/arkham-shard-embed/arkham_shard_embed/api.py +++ b/packages/arkham-shard-embed/arkham_shard_embed/api.py @@ -709,13 +709,63 @@ async def update_config(request: ConfigUpdateRequest): @router.get("/cache/stats") async def get_cache_stats(): - """Get embedding cache statistics.""" - if not _embedding_manager: - raise HTTPException(status_code=503, detail="Embedding service not initialized") + """ + Get embedding statistics. + Returns actual embedding counts from the vector store database, + which is more meaningful than in-memory cache stats since most + embeddings are processed through the worker pool. + """ try: - cache_info = _embedding_manager.get_cache_info() - return cache_info + # Get in-memory cache info if available + cache_info = {} + if _embedding_manager: + cache_info = _embedding_manager.get_cache_info() + + # Get actual embedding stats from database + total_embeddings = 0 + total_documents = 0 + total_chunks = 0 + + if _db_service: + try: + # Count total embeddings in vector store + embed_count = await _db_service.fetch_one( + "SELECT COUNT(*) as count FROM arkham_vectors.embeddings" + ) + total_embeddings = embed_count.get("count", 0) if embed_count else 0 + + # Count unique documents with embeddings + doc_count = await _db_service.fetch_one( + """SELECT COUNT(DISTINCT payload->>'document_id') as count + FROM arkham_vectors.embeddings + WHERE payload->>'document_id' IS NOT NULL""" + ) + total_documents = doc_count.get("count", 0) if doc_count else 0 + + # Count total chunks in system + chunk_count = await _db_service.fetch_one( + "SELECT COUNT(*) as count FROM arkham_frame.chunks" + ) + total_chunks = chunk_count.get("count", 0) if chunk_count else 0 + except Exception as db_err: + logger.debug(f"Could not get embedding counts from DB: {db_err}") + + # Calculate coverage rate (embeddings vs chunks) + coverage_rate = total_embeddings / total_chunks if total_chunks > 0 else 0.0 + + return { + "enabled": cache_info.get("enabled", False), + "hits": total_embeddings, # Total embeddings stored + "misses": max(0, total_chunks - total_embeddings), # Chunks not yet embedded + "size": total_documents, # Documents with embeddings + "max_size": total_chunks, # Total chunks available + "hit_rate": coverage_rate, # Embedding coverage percentage + # Additional context + "total_embeddings": total_embeddings, + "total_documents": total_documents, + "total_chunks": total_chunks, + } except Exception as e: logger.error(f"Failed to get cache stats: {e}", exc_info=True) raise HTTPException(status_code=500, detail=f"Failed to get cache stats: {str(e)}") diff --git a/packages/arkham-shard-graph/arkham_shard_graph/algorithms.py b/packages/arkham-shard-graph/arkham_shard_graph/algorithms.py index c16d785..eb80cd6 100644 --- a/packages/arkham-shard-graph/arkham_shard_graph/algorithms.py +++ b/packages/arkham-shard-graph/arkham_shard_graph/algorithms.py @@ -84,6 +84,317 @@ def find_shortest_path( return None + def find_all_paths( + self, + graph: Graph, + source_entity_id: str, + target_entity_id: str, + max_depth: int = 6, + max_paths: int = 10, + ) -> list[GraphPath]: + """ + Find all paths between two entities up to max_depth. + + Uses DFS with backtracking to enumerate paths. + + Args: + graph: Graph to search + source_entity_id: Source entity ID + target_entity_id: Target entity ID + max_depth: Maximum path length + max_paths: Maximum number of paths to return + + Returns: + List of GraphPath objects, shortest first + """ + adjacency = self._build_adjacency_dict(graph.edges) + all_paths: list[GraphPath] = [] + + def dfs(current: str, target: str, path: list[str], visited: set[str]): + if len(all_paths) >= max_paths: + return + + if len(path) > max_depth + 1: + return + + if current == target: + # Found a path + edges = self._get_path_edges(graph.edges, path) + total_weight = sum(e.weight for e in edges) + all_paths.append(GraphPath( + source_entity_id=source_entity_id, + target_entity_id=target_entity_id, + path=path.copy(), + edges=edges, + total_weight=total_weight, + path_length=len(path) - 1, + )) + return + + for neighbor in adjacency.get(current, []): + if neighbor not in visited: + visited.add(neighbor) + path.append(neighbor) + dfs(neighbor, target, path, visited) + path.pop() + visited.remove(neighbor) + + # Start DFS + dfs(source_entity_id, target_entity_id, [source_entity_id], {source_entity_id}) + + # Sort by path length (shortest first) + all_paths.sort(key=lambda p: p.path_length) + + return all_paths + + def find_weighted_path( + self, + graph: Graph, + source_entity_id: str, + target_entity_id: str, + max_depth: int = 10, + use_max_weight: bool = True, + ) -> GraphPath | None: + """ + Find optimal weighted path using Dijkstra's algorithm. + + Can find either maximum weight path (strongest connections) + or minimum weight path (weakest connections). + + Args: + graph: Graph to search + source_entity_id: Source entity ID + target_entity_id: Target entity ID + max_depth: Maximum path length + use_max_weight: If True, find path with maximum total weight + + Returns: + GraphPath if found, None otherwise + """ + import heapq + + # Build weighted adjacency + adjacency = self._build_weighted_adjacency(graph.edges) + + # For max weight, we negate weights and use min-heap + # For min weight, we use positive weights + def get_edge_cost(weight: float) -> float: + if use_max_weight: + return -weight # Negate for max-heap behavior + return weight + + # Dijkstra's algorithm + # heap entries: (cost, path_length, node_id, path) + heap = [(0.0, 0, source_entity_id, [source_entity_id])] + visited: dict[str, float] = {} + + while heap: + cost, path_len, current, path = heapq.heappop(heap) + + if path_len > max_depth: + continue + + if current in visited: + continue + visited[current] = cost + + if current == target_entity_id: + # Found target - reconstruct path + edges = self._get_path_edges(graph.edges, path) + total_weight = sum(e.weight for e in edges) + return GraphPath( + source_entity_id=source_entity_id, + target_entity_id=target_entity_id, + path=path, + edges=edges, + total_weight=total_weight, + path_length=len(path) - 1, + ) + + for neighbor, weight in adjacency.get(current, []): + if neighbor not in visited: + new_cost = cost + get_edge_cost(weight) + heapq.heappush( + heap, + (new_cost, path_len + 1, neighbor, path + [neighbor]) + ) + + return None + + def find_constrained_path( + self, + graph: Graph, + source_entity_id: str, + target_entity_id: str, + required_entities: list[str] | None = None, + excluded_entities: list[str] | None = None, + required_relationship_types: list[str] | None = None, + min_edge_weight: float = 0.0, + max_depth: int = 8, + ) -> GraphPath | None: + """ + Find path with constraints on nodes and edges. + + Args: + graph: Graph to search + source_entity_id: Source entity ID + target_entity_id: Target entity ID + required_entities: Entities that MUST be on the path + excluded_entities: Entities that MUST NOT be on the path + required_relationship_types: Only traverse these relationship types + min_edge_weight: Minimum weight for edges to traverse + max_depth: Maximum path length + + Returns: + GraphPath if found, None otherwise + """ + required = set(required_entities or []) + excluded = set(excluded_entities or []) + required_types = set(required_relationship_types) if required_relationship_types else None + + # Build filtered adjacency + adjacency: dict[str, list[tuple[str, float, str]]] = defaultdict(list) + for edge in graph.edges: + # Filter by relationship type + if required_types and edge.relationship_type not in required_types: + continue + # Filter by weight + if edge.weight < min_edge_weight: + continue + + adjacency[edge.source].append((edge.target, edge.weight, edge.relationship_type)) + adjacency[edge.target].append((edge.source, edge.weight, edge.relationship_type)) + + # Use BFS with required entity tracking + # State: (current_node, path, required_visited) + from collections import deque + + initial_required = required & {source_entity_id} + queue = deque([(source_entity_id, [source_entity_id], initial_required)]) + visited: dict[tuple[str, frozenset], bool] = {(source_entity_id, frozenset(initial_required)): True} + + while queue: + current, path, req_visited = queue.popleft() + + if len(path) > max_depth + 1: + continue + + # Check if we found the target with all required nodes visited + if current == target_entity_id and req_visited >= required: + edges = self._get_path_edges(graph.edges, path) + total_weight = sum(e.weight for e in edges) + return GraphPath( + source_entity_id=source_entity_id, + target_entity_id=target_entity_id, + path=path, + edges=edges, + total_weight=total_weight, + path_length=len(path) - 1, + ) + + for neighbor, weight, rel_type in adjacency.get(current, []): + # Skip excluded entities + if neighbor in excluded: + continue + + # Skip already visited in this path + if neighbor in path: + continue + + # Update required visited + new_req = req_visited | ({neighbor} & required) + state_key = (neighbor, frozenset(new_req)) + + if state_key not in visited: + visited[state_key] = True + queue.append((neighbor, path + [neighbor], new_req)) + + return None + + def find_paths_through( + self, + graph: Graph, + intermediate_entity_id: str, + max_sources: int = 5, + max_targets: int = 5, + max_depth: int = 3, + ) -> list[GraphPath]: + """ + Find paths that pass through a specific entity. + + Useful for finding what connections an entity bridges. + + Args: + graph: Graph to search + intermediate_entity_id: Entity that paths must pass through + max_sources: Maximum source entities to consider + max_targets: Maximum target entities to consider + max_depth: Maximum hops on each side of intermediate + + Returns: + List of paths passing through the intermediate entity + """ + adjacency = self._build_adjacency_dict(graph.edges) + + # Find entities reachable from intermediate (potential targets) + reachable_from: dict[str, int] = {} # entity_id -> distance + queue = deque([(intermediate_entity_id, 0)]) + visited = {intermediate_entity_id} + + while queue: + current, dist = queue.popleft() + if dist > max_depth: + continue + if current != intermediate_entity_id: + reachable_from[current] = dist + + for neighbor in adjacency.get(current, []): + if neighbor not in visited: + visited.add(neighbor) + queue.append((neighbor, dist + 1)) + + # Find entities that can reach intermediate (potential sources) + # (Same search since graph is undirected) + can_reach: dict[str, int] = reachable_from.copy() + + # Select top sources and targets by degree (most connected) + node_degrees = {n.id: n.degree for n in graph.nodes} + + source_candidates = sorted( + can_reach.keys(), + key=lambda x: node_degrees.get(x, 0), + reverse=True + )[:max_sources] + + target_candidates = sorted( + reachable_from.keys(), + key=lambda x: node_degrees.get(x, 0), + reverse=True + )[:max_targets] + + # Find paths between sources and targets that pass through intermediate + paths = [] + for source in source_candidates: + for target in target_candidates: + if source == target: + continue + + # Find path requiring intermediate + path = self.find_constrained_path( + graph, + source, + target, + required_entities=[intermediate_entity_id], + max_depth=max_depth * 2 + 1, + ) + if path: + paths.append(path) + + # Sort by total weight descending + paths.sort(key=lambda p: p.total_weight, reverse=True) + + return paths + def calculate_degree_centrality( self, graph: Graph, limit: int = 50 ) -> list[CentralityResult]: diff --git a/packages/arkham-shard-graph/arkham_shard_graph/api.py b/packages/arkham-shard-graph/arkham_shard_graph/api.py index 0d4f4c2..020be01 100644 --- a/packages/arkham-shard-graph/arkham_shard_graph/api.py +++ b/packages/arkham-shard-graph/arkham_shard_graph/api.py @@ -1949,6 +1949,241 @@ async def find_path(request: PathRequest) -> PathResponse: raise HTTPException(status_code=500, detail=str(e)) +# --- Advanced Pathfinding Endpoints --- + + +class AllPathsRequest(BaseModel): + """Request for finding all paths between entities.""" + project_id: str + source_entity_id: str + target_entity_id: str + max_depth: int = 6 + max_paths: int = 10 + + +class AllPathsResponse(BaseModel): + """Response for all paths query.""" + paths_found: int + paths: list[dict[str, Any]] + + +class WeightedPathRequest(BaseModel): + """Request for weighted path finding.""" + project_id: str + source_entity_id: str + target_entity_id: str + max_depth: int = 10 + use_max_weight: bool = True # True for strongest path, False for weakest + + +class ConstrainedPathRequest(BaseModel): + """Request for constrained path finding.""" + project_id: str + source_entity_id: str + target_entity_id: str + required_entities: list[str] | None = None + excluded_entities: list[str] | None = None + required_relationship_types: list[str] | None = None + min_edge_weight: float = 0.0 + max_depth: int = 8 + + +class PathsThroughRequest(BaseModel): + """Request for finding paths through an entity.""" + project_id: str + intermediate_entity_id: str + max_sources: int = 5 + max_targets: int = 5 + max_depth: int = 3 + + +@router.post("/paths/all") +async def find_all_paths(request: AllPathsRequest) -> AllPathsResponse: + """ + Find all paths between two entities. + + Returns multiple paths up to max_paths, sorted by length. + Useful for discovering all connection routes. + """ + if not _algorithms: + raise HTTPException(status_code=503, detail="Graph algorithms not available") + + try: + # Get graph + if _storage: + graph = await _storage.load_graph(request.project_id) + elif _builder: + graph = await _builder.build_graph(project_id=request.project_id) + else: + raise HTTPException(status_code=503, detail="Graph service not available") + + # Find all paths + paths = _algorithms.find_all_paths( + graph=graph, + source_entity_id=request.source_entity_id, + target_entity_id=request.target_entity_id, + max_depth=request.max_depth, + max_paths=request.max_paths, + ) + + return AllPathsResponse( + paths_found=len(paths), + paths=[p.to_dict() for p in paths], + ) + + except Exception as e: + logger.error(f"Error finding all paths: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=str(e)) + + +@router.post("/paths/weighted") +async def find_weighted_path(request: WeightedPathRequest) -> PathResponse: + """ + Find optimal weighted path between entities. + + Uses Dijkstra's algorithm to find the path with: + - Maximum total weight (strongest connections) when use_max_weight=True + - Minimum total weight (weakest connections) when use_max_weight=False + """ + if not _algorithms: + raise HTTPException(status_code=503, detail="Graph algorithms not available") + + try: + # Get graph + if _storage: + graph = await _storage.load_graph(request.project_id) + elif _builder: + graph = await _builder.build_graph(project_id=request.project_id) + else: + raise HTTPException(status_code=503, detail="Graph service not available") + + # Find weighted path + path = _algorithms.find_weighted_path( + graph=graph, + source_entity_id=request.source_entity_id, + target_entity_id=request.target_entity_id, + max_depth=request.max_depth, + use_max_weight=request.use_max_weight, + ) + + if not path: + return PathResponse( + path_found=False, + path_length=0, + path=[], + edges=[], + total_weight=0.0, + ) + + return PathResponse( + path_found=True, + path_length=path.path_length, + path=path.path, + edges=[e.to_dict() for e in path.edges], + total_weight=path.total_weight, + ) + + except Exception as e: + logger.error(f"Error finding weighted path: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=str(e)) + + +@router.post("/paths/constrained") +async def find_constrained_path(request: ConstrainedPathRequest) -> PathResponse: + """ + Find path with constraints on nodes and edges. + + Supports: + - Required entities (must pass through) + - Excluded entities (must avoid) + - Relationship type filtering + - Minimum edge weight + """ + if not _algorithms: + raise HTTPException(status_code=503, detail="Graph algorithms not available") + + try: + # Get graph + if _storage: + graph = await _storage.load_graph(request.project_id) + elif _builder: + graph = await _builder.build_graph(project_id=request.project_id) + else: + raise HTTPException(status_code=503, detail="Graph service not available") + + # Find constrained path + path = _algorithms.find_constrained_path( + graph=graph, + source_entity_id=request.source_entity_id, + target_entity_id=request.target_entity_id, + required_entities=request.required_entities, + excluded_entities=request.excluded_entities, + required_relationship_types=request.required_relationship_types, + min_edge_weight=request.min_edge_weight, + max_depth=request.max_depth, + ) + + if not path: + return PathResponse( + path_found=False, + path_length=0, + path=[], + edges=[], + total_weight=0.0, + ) + + return PathResponse( + path_found=True, + path_length=path.path_length, + path=path.path, + edges=[e.to_dict() for e in path.edges], + total_weight=path.total_weight, + ) + + except Exception as e: + logger.error(f"Error finding constrained path: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=str(e)) + + +@router.post("/paths/through") +async def find_paths_through_entity(request: PathsThroughRequest) -> AllPathsResponse: + """ + Find paths that pass through a specific entity. + + Useful for identifying what connections an entity bridges + and understanding their position in the network. + """ + if not _algorithms: + raise HTTPException(status_code=503, detail="Graph algorithms not available") + + try: + # Get graph + if _storage: + graph = await _storage.load_graph(request.project_id) + elif _builder: + graph = await _builder.build_graph(project_id=request.project_id) + else: + raise HTTPException(status_code=503, detail="Graph service not available") + + # Find paths through entity + paths = _algorithms.find_paths_through( + graph=graph, + intermediate_entity_id=request.intermediate_entity_id, + max_sources=request.max_sources, + max_targets=request.max_targets, + max_depth=request.max_depth, + ) + + return AllPathsResponse( + paths_found=len(paths), + paths=[p.to_dict() for p in paths], + ) + + except Exception as e: + logger.error(f"Error finding paths through entity: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=str(e)) + + @router.get("/centrality/{project_id}") async def calculate_centrality( project_id: str, diff --git a/packages/arkham-shard-parse/arkham_shard_parse/chunker.py b/packages/arkham-shard-parse/arkham_shard_parse/chunker.py index 763c7b7..96d23d1 100644 --- a/packages/arkham-shard-parse/arkham_shard_parse/chunker.py +++ b/packages/arkham-shard-parse/arkham_shard_parse/chunker.py @@ -126,9 +126,19 @@ def _chunk_by_sentences( Returns: List of text chunks """ - # Simple sentence splitting on period, exclamation, question mark + # Sentence splitting that preserves emails, URLs, and abbreviations + # Only splits on punctuation that looks like a sentence boundary: + # - Period/!/? followed by whitespace and then an uppercase letter + # - Period/!/? followed by newline + # Does NOT split on periods in emails, URLs, abbreviations, or decimals import re - sentences = re.split(r'[.!?]+', text) + + # Split on sentence-ending punctuation followed by whitespace and capital letter, + # or followed by newline/end of text + # This preserves: emails (agent.smith@cia.gov), URLs, abbreviations, decimals + sentence_pattern = r'(?<=[.!?])\s+(?=[A-Z])|(?<=[.!?])\s*(?=\n)' + + sentences = re.split(sentence_pattern, text) chunks = [] chunk_index = 0 @@ -210,8 +220,10 @@ def _chunk_semantic( import re import numpy as np - # Split into sentences - sentences = re.split(r'(?<=[.!?])\s+', text) + # Split into sentences - preserve emails, URLs, abbreviations + # Only split on punctuation followed by whitespace and capital letter + sentence_pattern = r'(?<=[.!?])\s+(?=[A-Z])|(?<=[.!?])\s*(?=\n)' + sentences = re.split(sentence_pattern, text) sentences = [s.strip() for s in sentences if s.strip()] if len(sentences) < 3: diff --git a/packages/arkham-shard-provenance/arkham_shard_provenance/api.py b/packages/arkham-shard-provenance/arkham_shard_provenance/api.py index 891e77a..df37375 100644 --- a/packages/arkham-shard-provenance/arkham_shard_provenance/api.py +++ b/packages/arkham-shard-provenance/arkham_shard_provenance/api.py @@ -18,18 +18,21 @@ _shard: Optional["ProvenanceShard"] = None _event_bus = None _storage = None +_forensic_analyzer = None def init_api( shard: "ProvenanceShard", event_bus, storage, + forensic_analyzer=None, ): """Initialize API with shard instance.""" - global _shard, _event_bus, _storage + global _shard, _event_bus, _storage, _forensic_analyzer _shard = shard _event_bus = event_bus _storage = storage + _forensic_analyzer = forensic_analyzer logger.info("Provenance API initialized") @@ -1048,3 +1051,435 @@ async def ai_junior_analyst(request: Request, body: AIJuniorAnalystRequest): "X-Accel-Buffering": "no", }, ) + + +# ============================================================================= +# Metadata Forensics Endpoints +# ============================================================================= + + +class ForensicScanRequest(BaseModel): + """Request to perform forensic scan on a document.""" + doc_id: str + + +class ForensicCompareRequest(BaseModel): + """Request to compare metadata between documents.""" + source_doc_id: str + target_doc_id: str + + +class ForensicScanResponse(BaseModel): + """Response from forensic scan.""" + scan: dict + + +class ForensicStatsResponse(BaseModel): + """Forensic statistics response.""" + stats: dict + + +@router.post("/forensics/scan", response_model=ForensicScanResponse) +async def scan_forensics( + request: Request, + body: ForensicScanRequest, +): + """ + Perform forensic metadata analysis on a document. + + Extracts and analyzes: + - EXIF data from images (camera info, GPS, timestamps) + - PDF metadata (author, creator, dates, XMP) + - Office document metadata (author, company, revision history) + + Performs integrity analysis: + - Timestamp consistency checks + - Metadata stripping detection + - Editing software detection + - Timeline reconstruction + + Args: + body: Request with doc_id + + Returns: + Complete forensic analysis results + """ + shard = get_shard(request) + + if not shard.forensic_analyzer: + raise HTTPException( + status_code=503, + detail="Forensic analyzer not available" + ) + + if not shard._db: + raise HTTPException(status_code=503, detail="Database not available") + + # Get document metadata and file path + doc_row = await shard._db.fetch_one( + """SELECT id, filename, storage_id, mime_type, file_size, metadata + FROM arkham_frame.documents WHERE id = :doc_id""", + {"doc_id": body.doc_id} + ) + + if not doc_row: + raise HTTPException(status_code=404, detail=f"Document {body.doc_id} not found") + + # Get storage path from storage_id or metadata + storage_id = doc_row.get("storage_id") + metadata = doc_row.get("metadata") or {} + if isinstance(metadata, str): + import json + try: + metadata = json.loads(metadata) + except json.JSONDecodeError: + metadata = {} + + storage_path = metadata.get("storage_path") + if not storage_id and not storage_path: + raise HTTPException( + status_code=400, + detail="Document has no associated file storage" + ) + + # Read file content using storage service + try: + if _storage and storage_id: + file_data = (await _storage.retrieve(storage_id))[0] + elif storage_path: + from pathlib import Path + file_data = Path(storage_path).read_bytes() + else: + raise ValueError("No storage path available") + except Exception as e: + raise HTTPException( + status_code=500, + detail=f"Failed to read file: {e}" + ) + + mime_type = doc_row.get("mime_type", "") + + # Perform forensic scan (use storage_path for file path reference) + file_path = storage_path or storage_id + scan_result = shard.forensic_analyzer.full_scan( + doc_id=body.doc_id, + file_path=file_path, + file_data=file_data, + mime_type=mime_type, + ) + + # Store the scan result + await shard._store_forensic_scan(scan_result) + + # Emit event + if _event_bus: + await _event_bus.emit( + "provenance.forensics.scanned", + { + "doc_id": body.doc_id, + "scan_id": scan_result.id, + "integrity_status": scan_result.integrity_status.value, + "findings_count": len(scan_result.findings), + }, + source="provenance-shard", + ) + + # Convert to response dict + scan_dict = { + "id": scan_result.id, + "doc_id": scan_result.doc_id, + "scan_status": scan_result.scan_status.value, + "file_hash_md5": scan_result.file_hash_md5, + "file_hash_sha256": scan_result.file_hash_sha256, + "file_hash_sha512": scan_result.file_hash_sha512, + "file_size": scan_result.file_size, + "exif_data": { + "make": scan_result.exif_data.make, + "model": scan_result.exif_data.model, + "serial_number": scan_result.exif_data.serial_number, + "datetime_original": scan_result.exif_data.datetime_original.isoformat() if scan_result.exif_data.datetime_original else None, + "datetime_digitized": scan_result.exif_data.datetime_digitized.isoformat() if scan_result.exif_data.datetime_digitized else None, + "datetime_modified": scan_result.exif_data.datetime_modified.isoformat() if scan_result.exif_data.datetime_modified else None, + "gps_latitude": scan_result.exif_data.gps_latitude, + "gps_longitude": scan_result.exif_data.gps_longitude, + "gps_altitude": scan_result.exif_data.gps_altitude, + "software": scan_result.exif_data.software, + "width": scan_result.exif_data.width, + "height": scan_result.exif_data.height, + } if scan_result.exif_data else None, + "pdf_metadata": { + "title": scan_result.pdf_metadata.title, + "author": scan_result.pdf_metadata.author, + "subject": scan_result.pdf_metadata.subject, + "creator": scan_result.pdf_metadata.creator, + "producer": scan_result.pdf_metadata.producer, + "creation_date": scan_result.pdf_metadata.creation_date.isoformat() if scan_result.pdf_metadata.creation_date else None, + "modification_date": scan_result.pdf_metadata.modification_date.isoformat() if scan_result.pdf_metadata.modification_date else None, + "keywords": scan_result.pdf_metadata.keywords, + "page_count": scan_result.pdf_metadata.page_count, + "pdf_version": scan_result.pdf_metadata.pdf_version, + "is_encrypted": scan_result.pdf_metadata.is_encrypted, + } if scan_result.pdf_metadata else None, + "office_metadata": { + "title": scan_result.office_metadata.title, + "author": scan_result.office_metadata.author, + "subject": scan_result.office_metadata.subject, + "company": scan_result.office_metadata.company, + "manager": scan_result.office_metadata.manager, + "created": scan_result.office_metadata.created.isoformat() if scan_result.office_metadata.created else None, + "modified": scan_result.office_metadata.modified.isoformat() if scan_result.office_metadata.modified else None, + "last_modified_by": scan_result.office_metadata.last_modified_by, + "revision": scan_result.office_metadata.revision, + "keywords": scan_result.office_metadata.keywords, + "category": scan_result.office_metadata.category, + } if scan_result.office_metadata else None, + "findings": [ + { + "finding_type": f.finding_type, + "severity": f.severity, + "description": f.description, + "evidence": f.evidence, + "confidence": f.confidence, + } + for f in scan_result.findings + ], + "integrity_status": scan_result.integrity_status.value, + "confidence_score": scan_result.confidence_score, + "timeline_events": [ + { + "id": e.id, + "doc_id": e.doc_id, + "event_type": e.event_type, + "event_timestamp": e.event_timestamp.isoformat() if e.event_timestamp else None, + "event_source": e.event_source, + "event_actor": e.event_actor, + "event_details": e.event_details, + "confidence": e.confidence, + "is_estimated": e.is_estimated, + } + for e in scan_result.timeline_events + ], + "scanned_at": scan_result.scanned_at.isoformat() if scan_result.scanned_at else None, + } + + return ForensicScanResponse(scan=scan_dict) + + +@router.post("/forensics/compare") +async def compare_forensics( + request: Request, + body: ForensicCompareRequest, +): + """ + Compare metadata between two documents. + + Analyzes similarities and differences: + - Hash matches (exact copies) + - Camera/device matches + - Author/creator matches + - Creation date proximity + + Determines relationship type: + - Copy: Exact hash match + - Same source: Same device/serial number + - Same author: Same author metadata + - Same camera: Same camera make/model + - Unrelated: No significant matches + + Args: + body: Request with source and target doc IDs + + Returns: + Comparison results with match score and relationship + """ + shard = get_shard(request) + + if not shard.forensic_analyzer: + raise HTTPException( + status_code=503, + detail="Forensic analyzer not available" + ) + + # Get existing scans for both documents + source_scans = await shard.get_document_forensic_scans(body.source_doc_id) + target_scans = await shard.get_document_forensic_scans(body.target_doc_id) + + if not source_scans: + raise HTTPException( + status_code=400, + detail=f"No forensic scan found for source document {body.source_doc_id}. Run /forensics/scan first." + ) + + if not target_scans: + raise HTTPException( + status_code=400, + detail=f"No forensic scan found for target document {body.target_doc_id}. Run /forensics/scan first." + ) + + # Use most recent scans + source_scan = source_scans[0] + target_scan = target_scans[0] + + # Build MetadataForensicScan objects from stored data + from .models import ( + MetadataForensicScan, ForensicScanStatus, IntegrityStatus, + ExifData, PdfMetadata, OfficeMetadata, + ) + from datetime import datetime + + def build_scan_from_dict(scan_dict: dict) -> MetadataForensicScan: + scan = MetadataForensicScan( + id=scan_dict.get("id", ""), + doc_id=scan_dict.get("doc_id", ""), + scan_status=ForensicScanStatus(scan_dict.get("scan_status", "completed")), + file_hash_md5=scan_dict.get("file_hash_md5"), + file_hash_sha256=scan_dict.get("file_hash_sha256"), + file_hash_sha512=scan_dict.get("file_hash_sha512"), + file_size=scan_dict.get("file_size"), + integrity_status=IntegrityStatus(scan_dict.get("integrity_status", "unknown")), + confidence_score=scan_dict.get("confidence_score", 0.0), + ) + + # Build EXIF data if present + exif = scan_dict.get("exif_data", {}) + if exif and any(exif.values()): + scan.exif_data = ExifData( + make=exif.get("make"), + model=exif.get("model"), + serial_number=exif.get("serial_number"), + software=exif.get("software"), + ) + + # Build PDF metadata if present + pdf = scan_dict.get("pdf_metadata", {}) + if pdf and any(pdf.values()): + scan.pdf_metadata = PdfMetadata( + title=pdf.get("title"), + author=pdf.get("author"), + creator=pdf.get("creator"), + producer=pdf.get("producer"), + ) + + # Build Office metadata if present + office = scan_dict.get("office_metadata", {}) + if office and any(office.values()): + scan.office_metadata = OfficeMetadata( + title=office.get("title"), + author=office.get("author"), + company=office.get("company"), + ) + + return scan + + source = build_scan_from_dict(source_scan) + target = build_scan_from_dict(target_scan) + + # Perform comparison + comparison = shard.forensic_analyzer.compare_documents(source, target) + + # Store comparison result + if shard._db: + import json + tenant_id = shard.get_tenant_id_or_none() + await shard._db.execute( + """ + INSERT INTO arkham_provenance.metadata_comparisons ( + id, source_doc_id, target_doc_id, comparison_type, + match_score, differences, similarities, relationship_type, + confidence, created_at, metadata, tenant_id + ) VALUES ( + :id, :source_doc_id, :target_doc_id, :comparison_type, + :match_score, :differences, :similarities, :relationship_type, + :confidence, :created_at, :metadata, :tenant_id + ) + """, + { + "id": comparison.id, + "source_doc_id": comparison.source_doc_id, + "target_doc_id": comparison.target_doc_id, + "comparison_type": comparison.comparison_type, + "match_score": comparison.match_score, + "differences": json.dumps(comparison.differences), + "similarities": json.dumps(comparison.similarities), + "relationship_type": comparison.relationship_type.value if comparison.relationship_type else None, + "confidence": comparison.confidence, + "created_at": comparison.created_at.isoformat() if comparison.created_at else datetime.utcnow().isoformat(), + "metadata": json.dumps(comparison.metadata), + "tenant_id": str(tenant_id) if tenant_id else None, + } + ) + + return { + "comparison": { + "id": comparison.id, + "source_doc_id": comparison.source_doc_id, + "target_doc_id": comparison.target_doc_id, + "comparison_type": comparison.comparison_type, + "match_score": comparison.match_score, + "differences": comparison.differences, + "similarities": comparison.similarities, + "relationship_type": comparison.relationship_type.value if comparison.relationship_type else None, + "confidence": comparison.confidence, + } + } + + +@router.get("/forensics/stats", response_model=ForensicStatsResponse) +async def get_forensic_stats(request: Request): + """ + Get forensic analysis statistics. + + Returns aggregated statistics about forensic scans: + - Total scans performed + - Scans by status + - Documents with findings + - Integrity status counts + - Average confidence score + """ + shard = get_shard(request) + + stats = await shard.get_forensic_stats() + return ForensicStatsResponse(stats=stats) + + +@router.get("/forensics/document/{doc_id}") +async def get_document_forensic_scans( + request: Request, + doc_id: str, +): + """ + Get all forensic scans for a document. + + Args: + doc_id: Document ID + + Returns: + List of scans for the document + """ + shard = get_shard(request) + + scans = await shard.get_document_forensic_scans(doc_id) + return {"scans": scans, "total": len(scans)} + + +@router.get("/forensics/{scan_id}") +async def get_forensic_scan( + request: Request, + scan_id: str, +): + """ + Get a specific forensic scan by ID. + + Args: + scan_id: Scan ID to retrieve + + Returns: + Scan details + """ + shard = get_shard(request) + + scan = await shard.get_forensic_scan(scan_id) + if not scan: + raise HTTPException(status_code=404, detail=f"Scan {scan_id} not found") + + return {"scan": scan} diff --git a/packages/arkham-shard-provenance/arkham_shard_provenance/forensics.py b/packages/arkham-shard-provenance/arkham_shard_provenance/forensics.py new file mode 100644 index 0000000..d038c9e --- /dev/null +++ b/packages/arkham-shard-provenance/arkham_shard_provenance/forensics.py @@ -0,0 +1,889 @@ +"""Metadata forensics module. + +Extracts and analyzes metadata from: +- Images (EXIF, IPTC, XMP) +- PDFs (Info dict, XMP) +- Office documents (Core properties) + +Performs integrity analysis: +- Timestamp consistency checks +- Metadata stripping detection +- Editing software detection +- Timeline reconstruction +""" + +import hashlib +import logging +import uuid +from datetime import datetime +from typing import Optional, Tuple + +from .models import ( + ExifData, + ForensicFinding, + ForensicScanStatus, + IntegrityStatus, + MetadataComparison, + MetadataForensicScan, + OfficeMetadata, + PdfMetadata, + RelationshipType, + TimelineEvent, +) + +logger = logging.getLogger(__name__) + + +class MetadataForensicAnalyzer: + """ + Comprehensive metadata forensic analyzer. + + Extracts and analyzes metadata from: + - Images (EXIF, IPTC, XMP via Pillow) + - PDFs (Info dict, XMP via pypdf) + - Office documents (Core properties via python-docx/openpyxl) + + Performs forensic analysis: + - Timestamp consistency validation + - Metadata integrity checks + - Editing software detection + - Timeline reconstruction from available metadata + """ + + def __init__(self): + self._magic = None + + def _get_magic(self): + """Lazy load python-magic with fallback.""" + if self._magic is None: + try: + import magic + self._magic = magic.Magic(mime=True) + except ImportError: + logger.warning("python-magic not available") + self._magic = False + except Exception as e: + logger.warning(f"libmagic not available: {e}") + self._magic = False + return self._magic if self._magic is not False else None + + def calculate_hashes(self, data: bytes) -> dict: + """ + Calculate multiple hash digests for integrity verification. + + Args: + data: Raw file bytes + + Returns: + Dict with md5, sha256, sha512 hashes + """ + return { + 'md5': hashlib.md5(data).hexdigest(), + 'sha256': hashlib.sha256(data).hexdigest(), + 'sha512': hashlib.sha512(data).hexdigest(), + } + + def extract_exif(self, file_path: str) -> ExifData: + """ + Extract EXIF data from image file. + + Extracts: + - Camera make/model/serial + - Date/time stamps (original, digitized, modified) + - GPS coordinates + - Software used + - Image dimensions + + Args: + file_path: Path to image file + + Returns: + ExifData with extracted fields + """ + exif = ExifData() + + try: + from PIL import Image + from PIL.ExifTags import TAGS, GPSTAGS, IFD + except ImportError: + logger.warning("Pillow not available - EXIF extraction disabled") + return exif + + try: + with Image.open(file_path) as img: + exif.width = img.width + exif.height = img.height + + exif_data = img.getexif() + if not exif_data: + return exif + + raw_data = {} + for tag_id, value in exif_data.items(): + tag_name = TAGS.get(tag_id, str(tag_id)) + # Convert bytes to string for JSON serialization + if isinstance(value, bytes): + try: + value = value.decode('utf-8', errors='ignore') + except Exception: + value = str(value) + raw_data[tag_name] = value + + exif.raw_data = raw_data + exif.make = raw_data.get('Make') + exif.model = raw_data.get('Model') + exif.software = raw_data.get('Software') + + # Parse DateTime fields + datetime_fields = [ + ('DateTimeOriginal', 'datetime_original'), + ('DateTimeDigitized', 'datetime_digitized'), + ('DateTime', 'datetime_modified'), + ] + for exif_field, attr in datetime_fields: + if exif_field in raw_data: + try: + dt = datetime.strptime( + raw_data[exif_field], + '%Y:%m:%d %H:%M:%S' + ) + setattr(exif, attr, dt) + except (ValueError, TypeError): + pass + + # Extract GPS data + try: + gps_ifd = exif_data.get_ifd(IFD.GPSInfo) + if gps_ifd: + # GPS Latitude + lat_data = gps_ifd.get(2) # GPSLatitude + lat_ref = gps_ifd.get(1) # GPSLatitudeRef + if lat_data and lat_ref: + exif.gps_latitude = self._convert_gps_coords(lat_data, lat_ref) + + # GPS Longitude + lon_data = gps_ifd.get(4) # GPSLongitude + lon_ref = gps_ifd.get(3) # GPSLongitudeRef + if lon_data and lon_ref: + exif.gps_longitude = self._convert_gps_coords(lon_data, lon_ref) + + # GPS Altitude + alt_data = gps_ifd.get(6) # GPSAltitude + if alt_data: + try: + exif.gps_altitude = float(alt_data) + except (ValueError, TypeError): + pass + except Exception as e: + logger.debug(f"GPS extraction failed: {e}") + + except Exception as e: + logger.warning(f"EXIF extraction failed for {file_path}: {e}") + + return exif + + def _convert_gps_coords( + self, + coords: Optional[Tuple], + ref: Optional[str] + ) -> Optional[float]: + """ + Convert GPS from EXIF format to decimal degrees. + + Args: + coords: Tuple of (degrees, minutes, seconds) or IFDRational values + ref: Reference direction (N/S/E/W) + + Returns: + Decimal degrees or None if conversion fails + """ + if not coords or not ref: + return None + + try: + # Handle IFDRational or plain numbers + def to_float(val): + if hasattr(val, 'numerator') and hasattr(val, 'denominator'): + return float(val.numerator) / float(val.denominator) + return float(val) + + degrees = to_float(coords[0]) + minutes = to_float(coords[1]) + seconds = to_float(coords[2]) + + decimal = degrees + minutes / 60 + seconds / 3600 + + if ref in ['S', 'W']: + decimal = -decimal + + return decimal + except (ValueError, IndexError, TypeError, ZeroDivisionError) as e: + logger.debug(f"GPS conversion failed: {e}") + return None + + def extract_pdf_metadata(self, file_path: str) -> PdfMetadata: + """ + Extract metadata from PDF file. + + Extracts: + - Document info (title, author, subject, keywords) + - Creator/Producer applications + - Creation/Modification dates + - XMP metadata + - Page count and PDF version + - Encryption status + + Args: + file_path: Path to PDF file + + Returns: + PdfMetadata with extracted fields + """ + pdf = PdfMetadata() + + try: + from pypdf import PdfReader + except ImportError: + logger.warning("pypdf not available - PDF extraction disabled") + return pdf + + try: + reader = PdfReader(file_path) + + # Basic document info + meta = reader.metadata + if meta: + pdf.title = getattr(meta, 'title', None) + pdf.author = getattr(meta, 'author', None) + pdf.subject = getattr(meta, 'subject', None) + pdf.creator = getattr(meta, 'creator', None) + pdf.producer = getattr(meta, 'producer', None) + + # Parse dates + creation_date = getattr(meta, 'creation_date', None) + if creation_date: + pdf.creation_date = creation_date + + modification_date = getattr(meta, 'modification_date', None) + if modification_date: + pdf.modification_date = modification_date + + pdf.page_count = len(reader.pages) + + # Get PDF version if available + try: + pdf.pdf_version = reader.pdf_header + except Exception: + pass + + # XMP metadata + try: + xmp = reader.xmp_metadata + if xmp: + xmp_data = {} + for attr in ['dc_title', 'dc_creator', 'dc_description', + 'xmp_create_date', 'xmp_modify_date', + 'pdf_producer', 'pdf_keywords']: + val = getattr(xmp, attr, None) + if val: + xmp_data[attr] = str(val) if not isinstance(val, (list, dict)) else val + pdf.xmp_data = xmp_data + except Exception as e: + logger.debug(f"XMP extraction failed: {e}") + + pdf.is_encrypted = reader.is_encrypted + + except Exception as e: + logger.warning(f"PDF metadata extraction failed for {file_path}: {e}") + + return pdf + + def extract_office_metadata(self, file_path: str) -> OfficeMetadata: + """ + Extract metadata from Office documents (docx, xlsx, pptx). + + Args: + file_path: Path to Office file + + Returns: + OfficeMetadata with extracted fields + """ + office = OfficeMetadata() + + # Try python-docx for Word documents + if file_path.lower().endswith('.docx'): + try: + from docx import Document + doc = Document(file_path) + props = doc.core_properties + + office.title = props.title + office.author = props.author + office.subject = props.subject + office.company = getattr(props, 'company', None) + office.created = props.created + office.modified = props.modified + office.last_modified_by = props.last_modified_by + office.revision = props.revision + office.category = props.category + office.comments = props.comments + if props.keywords: + office.keywords = [k.strip() for k in props.keywords.split(',')] + + return office + except ImportError: + logger.debug("python-docx not available") + except Exception as e: + logger.warning(f"DOCX extraction failed: {e}") + + # Generic ZIP-based Office Open XML extraction + try: + import zipfile + import xml.etree.ElementTree as ET + + with zipfile.ZipFile(file_path, 'r') as zf: + # Try to read docProps/core.xml + if 'docProps/core.xml' in zf.namelist(): + with zf.open('docProps/core.xml') as f: + tree = ET.parse(f) + root = tree.getroot() + + # Define namespaces + ns = { + 'cp': 'http://schemas.openxmlformats.org/package/2006/metadata/core-properties', + 'dc': 'http://purl.org/dc/elements/1.1/', + 'dcterms': 'http://purl.org/dc/terms/', + } + + # Extract properties + title = root.find('.//dc:title', ns) + if title is not None and title.text: + office.title = title.text + + creator = root.find('.//dc:creator', ns) + if creator is not None and creator.text: + office.author = creator.text + + subject = root.find('.//dc:subject', ns) + if subject is not None and subject.text: + office.subject = subject.text + + created = root.find('.//dcterms:created', ns) + if created is not None and created.text: + try: + office.created = datetime.fromisoformat( + created.text.replace('Z', '+00:00') + ) + except ValueError: + pass + + modified = root.find('.//dcterms:modified', ns) + if modified is not None and modified.text: + try: + office.modified = datetime.fromisoformat( + modified.text.replace('Z', '+00:00') + ) + except ValueError: + pass + + last_mod_by = root.find('.//cp:lastModifiedBy', ns) + if last_mod_by is not None and last_mod_by.text: + office.last_modified_by = last_mod_by.text + + revision = root.find('.//cp:revision', ns) + if revision is not None and revision.text: + try: + office.revision = int(revision.text) + except ValueError: + pass + + except Exception as e: + logger.debug(f"Office metadata extraction failed: {e}") + + return office + + def analyze_integrity( + self, + exif: Optional[ExifData] = None, + pdf: Optional[PdfMetadata] = None, + office: Optional[OfficeMetadata] = None, + ) -> Tuple[IntegrityStatus, list[ForensicFinding], float]: + """ + Analyze metadata for integrity issues. + + Checks: + - Timestamp consistency (creation vs modification dates) + - Missing expected metadata (EXIF stripped) + - Signs of editing software + - Revision count anomalies + + Args: + exif: EXIF data from image (optional) + pdf: PDF metadata (optional) + office: Office metadata (optional) + + Returns: + Tuple of (status, findings, confidence) + """ + findings = [] + suspicious_count = 0 + + if exif: + # Check for stripped EXIF (minimal metadata) + if not exif.raw_data or len(exif.raw_data) < 5: + findings.append(ForensicFinding( + finding_type="exif_minimal", + severity="medium", + description="Minimal or missing EXIF data - may have been stripped", + confidence=0.7, + )) + suspicious_count += 1 + + # Check timestamp inconsistencies + if exif.datetime_original and exif.datetime_digitized: + diff = abs((exif.datetime_original - exif.datetime_digitized).total_seconds()) + if diff > 60: # More than 1 minute difference + findings.append(ForensicFinding( + finding_type="timestamp_inconsistency", + severity="medium", + description=f"DateTime Original and Digitized differ by {diff:.0f} seconds", + evidence={ + "original": exif.datetime_original.isoformat(), + "digitized": exif.datetime_digitized.isoformat(), + }, + confidence=0.8, + )) + suspicious_count += 1 + + # Check for editing software + if exif.software: + editing_tools = ['photoshop', 'gimp', 'lightroom', 'snapseed', + 'picasa', 'afterlight', 'vsco'] + software_lower = exif.software.lower() + if any(tool in software_lower for tool in editing_tools): + findings.append(ForensicFinding( + finding_type="editing_software_detected", + severity="low", + description=f"Image edited with: {exif.software}", + evidence={"software": exif.software}, + confidence=1.0, + )) + + if pdf: + # Check creation vs modification date + if pdf.creation_date and pdf.modification_date: + if pdf.modification_date < pdf.creation_date: + findings.append(ForensicFinding( + finding_type="timestamp_anomaly", + severity="high", + description="Modification date is before creation date", + evidence={ + "creation": pdf.creation_date.isoformat() if pdf.creation_date else None, + "modification": pdf.modification_date.isoformat() if pdf.modification_date else None, + }, + confidence=0.95, + )) + suspicious_count += 2 + + # Check for PDF manipulation tools + if pdf.producer: + manipulation_tools = ['ghostscript', 'pdftk', 'pdf-tools', + 'itext', 'fpdf', 'reportlab'] + producer_lower = pdf.producer.lower() + if any(tool in producer_lower for tool in manipulation_tools): + findings.append(ForensicFinding( + finding_type="pdf_tool_detected", + severity="low", + description=f"PDF processed with: {pdf.producer}", + evidence={"producer": pdf.producer}, + confidence=0.9, + )) + + # Check if PDF is encrypted + if pdf.is_encrypted: + findings.append(ForensicFinding( + finding_type="pdf_encrypted", + severity="low", + description="PDF is encrypted - metadata may be limited", + confidence=1.0, + )) + + if office: + # Check creation vs modification date + if office.created and office.modified: + if office.modified < office.created: + findings.append(ForensicFinding( + finding_type="timestamp_anomaly", + severity="high", + description="Modification date is before creation date", + evidence={ + "creation": office.created.isoformat() if office.created else None, + "modification": office.modified.isoformat() if office.modified else None, + }, + confidence=0.95, + )) + suspicious_count += 2 + + # Check revision count + if office.revision is not None: + if office.revision == 1 and office.modified: + # Single revision but has modification date + if office.created and office.modified != office.created: + findings.append(ForensicFinding( + finding_type="revision_anomaly", + severity="medium", + description="Document shows only 1 revision but has different created/modified dates", + evidence={"revision": office.revision}, + confidence=0.7, + )) + suspicious_count += 1 + elif office.revision > 100: + findings.append(ForensicFinding( + finding_type="high_revision_count", + severity="low", + description=f"Document has {office.revision} revisions - heavily edited", + evidence={"revision": office.revision}, + confidence=0.9, + )) + + # Determine status + if suspicious_count >= 3: + status = IntegrityStatus.TAMPERED + elif suspicious_count >= 1: + status = IntegrityStatus.SUSPICIOUS + elif findings: + status = IntegrityStatus.CLEAN # Has findings but not suspicious + else: + status = IntegrityStatus.CLEAN + + # Calculate confidence + if findings: + confidence = sum(f.confidence for f in findings) / len(findings) + else: + confidence = 1.0 # High confidence in clean status if no findings + + return status, findings, confidence + + def build_timeline( + self, + doc_id: str, + exif: Optional[ExifData] = None, + pdf: Optional[PdfMetadata] = None, + office: Optional[OfficeMetadata] = None, + ) -> list[TimelineEvent]: + """ + Build document timeline from all available metadata. + + Reconstructs the chronological history of a document based on + metadata timestamps from various sources. + + Args: + doc_id: Document identifier + exif: EXIF data (optional) + pdf: PDF metadata (optional) + office: Office metadata (optional) + + Returns: + List of timeline events, sorted chronologically + """ + events = [] + + if exif: + if exif.datetime_original: + events.append(TimelineEvent( + id=str(uuid.uuid4()), + doc_id=doc_id, + event_type="captured", + event_timestamp=exif.datetime_original, + event_source="exif", + event_actor=f"{exif.make or ''} {exif.model or ''}".strip() or None, + event_details={"field": "DateTimeOriginal"}, + )) + + if exif.datetime_digitized and exif.datetime_digitized != exif.datetime_original: + events.append(TimelineEvent( + id=str(uuid.uuid4()), + doc_id=doc_id, + event_type="digitized", + event_timestamp=exif.datetime_digitized, + event_source="exif", + event_details={"field": "DateTimeDigitized"}, + )) + + if exif.datetime_modified: + events.append(TimelineEvent( + id=str(uuid.uuid4()), + doc_id=doc_id, + event_type="modified", + event_timestamp=exif.datetime_modified, + event_source="exif", + event_actor=exif.software, + event_details={"field": "DateTime"}, + )) + + if pdf: + if pdf.creation_date: + events.append(TimelineEvent( + id=str(uuid.uuid4()), + doc_id=doc_id, + event_type="created", + event_timestamp=pdf.creation_date, + event_source="pdf", + event_actor=pdf.creator, + event_details={"creator": pdf.creator, "producer": pdf.producer}, + )) + + if pdf.modification_date and pdf.modification_date != pdf.creation_date: + events.append(TimelineEvent( + id=str(uuid.uuid4()), + doc_id=doc_id, + event_type="modified", + event_timestamp=pdf.modification_date, + event_source="pdf", + event_actor=pdf.producer, + event_details={"producer": pdf.producer}, + )) + + if office: + if office.created: + events.append(TimelineEvent( + id=str(uuid.uuid4()), + doc_id=doc_id, + event_type="created", + event_timestamp=office.created, + event_source="office", + event_actor=office.author, + event_details={"author": office.author, "company": office.company}, + )) + + if office.modified and office.modified != office.created: + events.append(TimelineEvent( + id=str(uuid.uuid4()), + doc_id=doc_id, + event_type="modified", + event_timestamp=office.modified, + event_source="office", + event_actor=office.last_modified_by, + event_details={ + "last_modified_by": office.last_modified_by, + "revision": office.revision, + }, + )) + + # Sort by timestamp + events.sort(key=lambda e: e.event_timestamp or datetime.min) + return events + + def compare_documents( + self, + scan1: MetadataForensicScan, + scan2: MetadataForensicScan, + ) -> MetadataComparison: + """ + Compare metadata between two documents. + + Analyzes: + - Hash matches (exact copies) + - Camera/device matches + - Author/creator matches + - Creation date proximity + + Args: + scan1: First document's forensic scan + scan2: Second document's forensic scan + + Returns: + MetadataComparison with match score and relationship type + """ + comparison = MetadataComparison( + id=str(uuid.uuid4()), + source_doc_id=scan1.doc_id, + target_doc_id=scan2.doc_id, + comparison_type="comprehensive", + ) + + similarities = [] + differences = [] + + # Hash comparison (exact match) + if scan1.file_hash_sha256 and scan2.file_hash_sha256: + if scan1.file_hash_sha256 == scan2.file_hash_sha256: + similarities.append({ + "type": "exact_hash_match", + "field": "sha256", + "confidence": 1.0, + }) + comparison.relationship_type = RelationshipType.COPY + comparison.confidence = 1.0 + comparison.match_score = 1.0 + else: + differences.append({ + "type": "hash_mismatch", + "field": "sha256", + }) + + # EXIF comparison + if scan1.exif_data and scan2.exif_data: + exif1, exif2 = scan1.exif_data, scan2.exif_data + + # Camera match + if (exif1.make and exif2.make and exif1.make == exif2.make and + exif1.model and exif2.model and exif1.model == exif2.model): + similarities.append({ + "type": "same_camera", + "value": f"{exif1.make} {exif1.model}", + "confidence": 0.9, + }) + if not comparison.relationship_type: + comparison.relationship_type = RelationshipType.SAME_CAMERA + comparison.confidence = max(comparison.confidence, 0.7) + + # Serial number match (very strong indicator) + if exif1.serial_number and exif2.serial_number: + if exif1.serial_number == exif2.serial_number: + similarities.append({ + "type": "same_device", + "value": exif1.serial_number, + "confidence": 0.99, + }) + if not comparison.relationship_type: + comparison.relationship_type = RelationshipType.SAME_SOURCE + comparison.confidence = max(comparison.confidence, 0.95) + + # PDF comparison + if scan1.pdf_metadata and scan2.pdf_metadata: + pdf1, pdf2 = scan1.pdf_metadata, scan2.pdf_metadata + + # Author match + if pdf1.author and pdf2.author and pdf1.author == pdf2.author: + similarities.append({ + "type": "same_author", + "value": pdf1.author, + "confidence": 0.8, + }) + if not comparison.relationship_type: + comparison.relationship_type = RelationshipType.SAME_AUTHOR + comparison.confidence = max(comparison.confidence, 0.6) + + # Creator software match + if pdf1.creator and pdf2.creator and pdf1.creator == pdf2.creator: + similarities.append({ + "type": "same_creator_software", + "value": pdf1.creator, + "confidence": 0.5, + }) + + # Office metadata comparison + if scan1.office_metadata and scan2.office_metadata: + off1, off2 = scan1.office_metadata, scan2.office_metadata + + if off1.author and off2.author and off1.author == off2.author: + similarities.append({ + "type": "same_author", + "value": off1.author, + "confidence": 0.8, + }) + if not comparison.relationship_type: + comparison.relationship_type = RelationshipType.SAME_AUTHOR + comparison.confidence = max(comparison.confidence, 0.6) + + if off1.company and off2.company and off1.company == off2.company: + similarities.append({ + "type": "same_company", + "value": off1.company, + "confidence": 0.6, + }) + + comparison.similarities = similarities + comparison.differences = differences + + if similarities and not comparison.match_score: + comparison.match_score = sum( + s.get('confidence', 0.5) for s in similarities + ) / len(similarities) + + if not comparison.relationship_type: + comparison.relationship_type = RelationshipType.UNRELATED + + return comparison + + def full_scan( + self, + doc_id: str, + file_path: str, + file_data: bytes, + mime_type: str, + ) -> MetadataForensicScan: + """ + Perform complete forensic scan on a document. + + Extracts all available metadata, calculates file hashes, + performs integrity analysis, and reconstructs timeline. + + Args: + doc_id: Document identifier + file_path: Path to file on disk + file_data: Raw file bytes + mime_type: MIME type + + Returns: + Complete forensic scan results + """ + scan = MetadataForensicScan( + id=str(uuid.uuid4()), + doc_id=doc_id, + ) + + try: + # Calculate hashes + hashes = self.calculate_hashes(file_data) + scan.file_hash_md5 = hashes['md5'] + scan.file_hash_sha256 = hashes['sha256'] + scan.file_hash_sha512 = hashes['sha512'] + scan.file_size = len(file_data) + + # Extract metadata based on file type + mime_lower = mime_type.lower() if mime_type else "" + + if 'image' in mime_lower: + scan.exif_data = self.extract_exif(file_path) + + if 'pdf' in mime_lower: + scan.pdf_metadata = self.extract_pdf_metadata(file_path) + + if any(x in mime_lower for x in ['word', 'document', 'spreadsheet', + 'presentation', 'officedocument']): + scan.office_metadata = self.extract_office_metadata(file_path) + + # Also try Office extraction for common extensions + if file_path.lower().endswith(('.docx', '.xlsx', '.pptx')): + if not scan.office_metadata or not scan.office_metadata.author: + scan.office_metadata = self.extract_office_metadata(file_path) + + # Analyze integrity + status, findings, confidence = self.analyze_integrity( + exif=scan.exif_data, + pdf=scan.pdf_metadata, + office=scan.office_metadata, + ) + scan.integrity_status = status + scan.findings = findings + scan.confidence_score = confidence + + # Build timeline + scan.timeline_events = self.build_timeline( + doc_id=doc_id, + exif=scan.exif_data, + pdf=scan.pdf_metadata, + office=scan.office_metadata, + ) + + scan.scan_status = ForensicScanStatus.COMPLETED + scan.scanned_at = datetime.utcnow() + + except Exception as e: + logger.error(f"Forensic scan failed for {doc_id}: {e}", exc_info=True) + scan.scan_status = ForensicScanStatus.FAILED + scan.metadata["error"] = str(e) + + return scan diff --git a/packages/arkham-shard-provenance/arkham_shard_provenance/models.py b/packages/arkham-shard-provenance/arkham_shard_provenance/models.py index c49b7c4..02a18c1 100644 --- a/packages/arkham-shard-provenance/arkham_shard_provenance/models.py +++ b/packages/arkham-shard-provenance/arkham_shard_provenance/models.py @@ -248,3 +248,180 @@ class ChainExport: include_metadata: bool = True include_audit: bool = True include_verification: bool = True + + +# ============================================================================= +# Metadata Forensics Models +# ============================================================================= + + +class ForensicScanStatus(Enum): + """Status of a forensic scan.""" + PENDING = "pending" + COMPLETED = "completed" + FAILED = "failed" + PARTIAL = "partial" + + +class IntegrityStatus(Enum): + """Integrity status of a document.""" + CLEAN = "clean" + SUSPICIOUS = "suspicious" + TAMPERED = "tampered" + UNKNOWN = "unknown" + + +class RelationshipType(Enum): + """Type of relationship between documents based on metadata.""" + COPY = "copy" + DERIVATIVE = "derivative" + SAME_SOURCE = "same_source" + SAME_AUTHOR = "same_author" + SAME_CAMERA = "same_camera" + UNRELATED = "unrelated" + + +@dataclass +class ExifData: + """EXIF metadata from images.""" + make: Optional[str] = None + model: Optional[str] = None + serial_number: Optional[str] = None + lens_model: Optional[str] = None + + datetime_original: Optional[datetime] = None + datetime_digitized: Optional[datetime] = None + datetime_modified: Optional[datetime] = None + + gps_latitude: Optional[float] = None + gps_longitude: Optional[float] = None + gps_altitude: Optional[float] = None + + software: Optional[str] = None + width: Optional[int] = None + height: Optional[int] = None + + raw_data: Dict[str, Any] = field(default_factory=dict) + + +@dataclass +class PdfMetadata: + """PDF document metadata.""" + title: Optional[str] = None + author: Optional[str] = None + subject: Optional[str] = None + creator: Optional[str] = None + producer: Optional[str] = None + creation_date: Optional[datetime] = None + modification_date: Optional[datetime] = None + keywords: List[str] = field(default_factory=list) + xmp_data: Dict[str, Any] = field(default_factory=dict) + page_count: int = 0 + pdf_version: Optional[str] = None + is_encrypted: bool = False + + +@dataclass +class OfficeMetadata: + """Office document (docx, xlsx, pptx) metadata.""" + title: Optional[str] = None + author: Optional[str] = None + subject: Optional[str] = None + company: Optional[str] = None + manager: Optional[str] = None + created: Optional[datetime] = None + modified: Optional[datetime] = None + last_modified_by: Optional[str] = None + revision: Optional[int] = None + keywords: List[str] = field(default_factory=list) + category: Optional[str] = None + comments: Optional[str] = None + raw_data: Dict[str, Any] = field(default_factory=dict) + + +@dataclass +class ForensicFinding: + """A forensic analysis finding.""" + finding_type: str # timestamp_inconsistency, metadata_stripped, etc. + severity: str # low, medium, high, critical + description: str + evidence: Dict[str, Any] = field(default_factory=dict) + confidence: float = 1.0 + + +@dataclass +class TimelineEvent: + """An event in document timeline reconstructed from metadata.""" + id: str + doc_id: str + event_type: str # created, modified, accessed, printed + event_timestamp: Optional[datetime] = None + event_source: str = "" # exif, pdf, office, filesystem + event_actor: Optional[str] = None + event_details: Dict[str, Any] = field(default_factory=dict) + confidence: float = 1.0 + is_estimated: bool = False + + +@dataclass +class MetadataForensicScan: + """Complete forensic scan result.""" + id: str + doc_id: str + scan_status: ForensicScanStatus = ForensicScanStatus.PENDING + + # File hashes + file_hash_md5: Optional[str] = None + file_hash_sha256: Optional[str] = None + file_hash_sha512: Optional[str] = None + file_size: Optional[int] = None + + # Extracted metadata by type + exif_data: Optional[ExifData] = None + pdf_metadata: Optional[PdfMetadata] = None + office_metadata: Optional[OfficeMetadata] = None + + # Forensic findings + findings: List[ForensicFinding] = field(default_factory=list) + integrity_status: IntegrityStatus = IntegrityStatus.UNKNOWN + confidence_score: float = 0.0 + + # Reconstructed timeline + timeline_events: List[TimelineEvent] = field(default_factory=list) + + # Metadata + scanned_at: datetime = field(default_factory=datetime.utcnow) + metadata: Dict[str, Any] = field(default_factory=dict) + + +@dataclass +class MetadataComparison: + """Comparison between two documents' metadata.""" + id: str + source_doc_id: str + target_doc_id: str + comparison_type: str + + match_score: float = 0.0 + differences: List[Dict[str, Any]] = field(default_factory=list) + similarities: List[Dict[str, Any]] = field(default_factory=list) + + relationship_type: Optional[RelationshipType] = None + confidence: float = 0.0 + + created_at: datetime = field(default_factory=datetime.utcnow) + created_by: Optional[str] = None + metadata: Dict[str, Any] = field(default_factory=dict) + + +@dataclass +class ForensicStats: + """Statistics about forensic scans.""" + total_scans: int = 0 + scans_by_status: Dict[str, int] = field(default_factory=dict) + documents_with_findings: int = 0 + integrity_clean: int = 0 + integrity_suspicious: int = 0 + integrity_tampered: int = 0 + average_confidence: float = 0.0 + calculated_at: datetime = field(default_factory=datetime.utcnow) diff --git a/packages/arkham-shard-provenance/arkham_shard_provenance/shard.py b/packages/arkham-shard-provenance/arkham_shard_provenance/shard.py index d8fb4d9..d603cf4 100644 --- a/packages/arkham-shard-provenance/arkham_shard_provenance/shard.py +++ b/packages/arkham-shard-provenance/arkham_shard_provenance/shard.py @@ -1,11 +1,14 @@ """Provenance Shard - Evidence Chain Tracking and Data Lineage.""" +import json import logging from typing import Any, Dict, List, Optional from arkham_frame.shard_interface import ArkhamShard from .api import init_api, router +from .forensics import MetadataForensicAnalyzer +from .models import MetadataForensicScan, ForensicScanStatus logger = logging.getLogger(__name__) @@ -58,6 +61,9 @@ def __init__(self): self._lineage_tracker = None self._audit_logger = None + # Forensic analyzer + self.forensic_analyzer: MetadataForensicAnalyzer | None = None + async def initialize(self, frame) -> None: """ Initialize the Provenance shard with Frame services. @@ -87,11 +93,16 @@ async def initialize(self, frame) -> None: # Create database schema await self._create_schema() + # Initialize forensic analyzer + self.forensic_analyzer = MetadataForensicAnalyzer() + logger.info("Forensic metadata analyzer initialized") + # Initialize API with shard instance (shard implements all managers) init_api( shard=self, event_bus=self._event_bus, storage=self._storage, + forensic_analyzer=self.forensic_analyzer, ) # Subscribe to events for automatic tracking @@ -421,6 +432,124 @@ async def _create_schema(self) -> None: ON arkham_provenance_audit(tenant_id) """) + # =========================================== + # Metadata Forensics Tables + # =========================================== + + # Create schema for forensics tables + await self._db.execute(""" + CREATE SCHEMA IF NOT EXISTS arkham_provenance + """) + + # Forensic scans table + await self._db.execute(""" + CREATE TABLE IF NOT EXISTS arkham_provenance.forensic_scans ( + id TEXT PRIMARY KEY, + doc_id TEXT NOT NULL, + scan_status TEXT DEFAULT 'pending', + file_hash_md5 TEXT, + file_hash_sha256 TEXT, + file_hash_sha512 TEXT, + file_size BIGINT, + exif_data JSONB DEFAULT '{}', + pdf_metadata JSONB DEFAULT '{}', + office_metadata JSONB DEFAULT '{}', + findings JSONB DEFAULT '[]', + integrity_status TEXT DEFAULT 'unknown', + confidence_score REAL DEFAULT 0.0, + timeline_events JSONB DEFAULT '[]', + scanned_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + metadata JSONB DEFAULT '{}', + tenant_id UUID + ) + """) + + # Metadata comparisons table + await self._db.execute(""" + CREATE TABLE IF NOT EXISTS arkham_provenance.metadata_comparisons ( + id TEXT PRIMARY KEY, + source_doc_id TEXT NOT NULL, + target_doc_id TEXT NOT NULL, + comparison_type TEXT NOT NULL, + match_score REAL DEFAULT 0.0, + differences JSONB DEFAULT '[]', + similarities JSONB DEFAULT '[]', + relationship_type TEXT, + confidence REAL DEFAULT 0.0, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + created_by TEXT, + metadata JSONB DEFAULT '{}', + tenant_id UUID + ) + """) + + # Forensic timeline events (denormalized for fast queries) + await self._db.execute(""" + CREATE TABLE IF NOT EXISTS arkham_provenance.forensic_timeline ( + id TEXT PRIMARY KEY, + doc_id TEXT NOT NULL, + scan_id TEXT NOT NULL, + event_type TEXT NOT NULL, + event_timestamp TIMESTAMP, + event_source TEXT, + event_actor TEXT, + event_details JSONB DEFAULT '{}', + confidence REAL DEFAULT 1.0, + is_estimated BOOLEAN DEFAULT false, + tenant_id UUID + ) + """) + + # Indexes for forensics tables + await self._db.execute(""" + CREATE INDEX IF NOT EXISTS idx_forensic_scans_doc + ON arkham_provenance.forensic_scans(doc_id) + """) + await self._db.execute(""" + CREATE INDEX IF NOT EXISTS idx_forensic_scans_status + ON arkham_provenance.forensic_scans(scan_status) + """) + await self._db.execute(""" + CREATE INDEX IF NOT EXISTS idx_forensic_scans_integrity + ON arkham_provenance.forensic_scans(integrity_status) + """) + await self._db.execute(""" + CREATE INDEX IF NOT EXISTS idx_forensic_scans_hash + ON arkham_provenance.forensic_scans(file_hash_sha256) + """) + await self._db.execute(""" + CREATE INDEX IF NOT EXISTS idx_forensic_scans_tenant + ON arkham_provenance.forensic_scans(tenant_id) + """) + await self._db.execute(""" + CREATE INDEX IF NOT EXISTS idx_metadata_comparisons_source + ON arkham_provenance.metadata_comparisons(source_doc_id) + """) + await self._db.execute(""" + CREATE INDEX IF NOT EXISTS idx_metadata_comparisons_target + ON arkham_provenance.metadata_comparisons(target_doc_id) + """) + await self._db.execute(""" + CREATE INDEX IF NOT EXISTS idx_metadata_comparisons_tenant + ON arkham_provenance.metadata_comparisons(tenant_id) + """) + await self._db.execute(""" + CREATE INDEX IF NOT EXISTS idx_forensic_timeline_doc + ON arkham_provenance.forensic_timeline(doc_id) + """) + await self._db.execute(""" + CREATE INDEX IF NOT EXISTS idx_forensic_timeline_scan + ON arkham_provenance.forensic_timeline(scan_id) + """) + await self._db.execute(""" + CREATE INDEX IF NOT EXISTS idx_forensic_timeline_timestamp + ON arkham_provenance.forensic_timeline(event_timestamp) + """) + await self._db.execute(""" + CREATE INDEX IF NOT EXISTS idx_forensic_timeline_tenant + ON arkham_provenance.forensic_timeline(tenant_id) + """) + logger.info("Provenance database schema created") def _parse_jsonb(self, value: Any, default: Any = None) -> Any: @@ -2631,3 +2760,298 @@ async def verify_chain(self, chain_id: str) -> Dict[str, Any]: Verification result with status and details """ return await self.verify_chain_impl(chain_id) + + # === Forensic Analysis Methods === + + async def _store_forensic_scan(self, scan: MetadataForensicScan) -> None: + """ + Store a forensic scan result in the database. + + Args: + scan: MetadataForensicScan result to store + """ + if not self._db: + return + + from datetime import datetime + + tenant_id = self.get_tenant_id_or_none() + + # Convert dataclass objects to JSON dicts + exif_data = {} + if scan.exif_data: + exif_data = { + "make": scan.exif_data.make, + "model": scan.exif_data.model, + "serial_number": scan.exif_data.serial_number, + "lens_model": scan.exif_data.lens_model, + "datetime_original": scan.exif_data.datetime_original.isoformat() if scan.exif_data.datetime_original else None, + "datetime_digitized": scan.exif_data.datetime_digitized.isoformat() if scan.exif_data.datetime_digitized else None, + "datetime_modified": scan.exif_data.datetime_modified.isoformat() if scan.exif_data.datetime_modified else None, + "gps_latitude": scan.exif_data.gps_latitude, + "gps_longitude": scan.exif_data.gps_longitude, + "gps_altitude": scan.exif_data.gps_altitude, + "software": scan.exif_data.software, + "width": scan.exif_data.width, + "height": scan.exif_data.height, + "raw_data": scan.exif_data.raw_data, + } + + pdf_metadata = {} + if scan.pdf_metadata: + pdf_metadata = { + "title": scan.pdf_metadata.title, + "author": scan.pdf_metadata.author, + "subject": scan.pdf_metadata.subject, + "creator": scan.pdf_metadata.creator, + "producer": scan.pdf_metadata.producer, + "creation_date": scan.pdf_metadata.creation_date.isoformat() if scan.pdf_metadata.creation_date else None, + "modification_date": scan.pdf_metadata.modification_date.isoformat() if scan.pdf_metadata.modification_date else None, + "keywords": scan.pdf_metadata.keywords, + "xmp_data": scan.pdf_metadata.xmp_data, + "page_count": scan.pdf_metadata.page_count, + "pdf_version": scan.pdf_metadata.pdf_version, + "is_encrypted": scan.pdf_metadata.is_encrypted, + } + + office_metadata = {} + if scan.office_metadata: + office_metadata = { + "title": scan.office_metadata.title, + "author": scan.office_metadata.author, + "subject": scan.office_metadata.subject, + "company": scan.office_metadata.company, + "manager": scan.office_metadata.manager, + "created": scan.office_metadata.created.isoformat() if scan.office_metadata.created else None, + "modified": scan.office_metadata.modified.isoformat() if scan.office_metadata.modified else None, + "last_modified_by": scan.office_metadata.last_modified_by, + "revision": scan.office_metadata.revision, + "keywords": scan.office_metadata.keywords, + "category": scan.office_metadata.category, + "comments": scan.office_metadata.comments, + "raw_data": scan.office_metadata.raw_data, + } + + findings = [ + { + "finding_type": f.finding_type, + "severity": f.severity, + "description": f.description, + "evidence": f.evidence, + "confidence": f.confidence, + } + for f in scan.findings + ] + + timeline_events = [ + { + "id": e.id, + "doc_id": e.doc_id, + "event_type": e.event_type, + "event_timestamp": e.event_timestamp.isoformat() if e.event_timestamp else None, + "event_source": e.event_source, + "event_actor": e.event_actor, + "event_details": e.event_details, + "confidence": e.confidence, + "is_estimated": e.is_estimated, + } + for e in scan.timeline_events + ] + + await self._db.execute( + """ + INSERT INTO arkham_provenance.forensic_scans ( + id, doc_id, scan_status, file_hash_md5, file_hash_sha256, file_hash_sha512, + file_size, exif_data, pdf_metadata, office_metadata, findings, + integrity_status, confidence_score, timeline_events, scanned_at, metadata, tenant_id + ) VALUES ( + :id, :doc_id, :scan_status, :file_hash_md5, :file_hash_sha256, :file_hash_sha512, + :file_size, :exif_data, :pdf_metadata, :office_metadata, :findings, + :integrity_status, :confidence_score, :timeline_events, :scanned_at, :metadata, :tenant_id + ) + """, + { + "id": scan.id, + "doc_id": scan.doc_id, + "scan_status": scan.scan_status.value, + "file_hash_md5": scan.file_hash_md5, + "file_hash_sha256": scan.file_hash_sha256, + "file_hash_sha512": scan.file_hash_sha512, + "file_size": scan.file_size, + "exif_data": json.dumps(exif_data), + "pdf_metadata": json.dumps(pdf_metadata), + "office_metadata": json.dumps(office_metadata), + "findings": json.dumps(findings), + "integrity_status": scan.integrity_status.value, + "confidence_score": scan.confidence_score, + "timeline_events": json.dumps(timeline_events), + "scanned_at": scan.scanned_at.isoformat() if scan.scanned_at else datetime.utcnow().isoformat(), + "metadata": json.dumps(scan.metadata), + "tenant_id": str(tenant_id) if tenant_id else None, + } + ) + + # Also store timeline events in denormalized table for fast queries + for event in scan.timeline_events: + await self._db.execute( + """ + INSERT INTO arkham_provenance.forensic_timeline ( + id, doc_id, scan_id, event_type, event_timestamp, + event_source, event_actor, event_details, confidence, is_estimated, tenant_id + ) VALUES ( + :id, :doc_id, :scan_id, :event_type, :event_timestamp, + :event_source, :event_actor, :event_details, :confidence, :is_estimated, :tenant_id + ) + """, + { + "id": event.id, + "doc_id": event.doc_id, + "scan_id": scan.id, + "event_type": event.event_type, + "event_timestamp": event.event_timestamp.isoformat() if event.event_timestamp else None, + "event_source": event.event_source, + "event_actor": event.event_actor, + "event_details": json.dumps(event.event_details), + "confidence": event.confidence, + "is_estimated": event.is_estimated, + "tenant_id": str(tenant_id) if tenant_id else None, + } + ) + + async def get_forensic_scan(self, scan_id: str) -> Optional[Dict[str, Any]]: + """ + Get a forensic scan by ID. + + Args: + scan_id: Scan ID + + Returns: + Scan data dict or None + """ + if not self._db: + return None + + tenant_id = self.get_tenant_id_or_none() + query = "SELECT * FROM arkham_provenance.forensic_scans WHERE id = :id" + params: Dict[str, Any] = {"id": scan_id} + + if tenant_id: + query += " AND tenant_id = :tenant_id" + params["tenant_id"] = str(tenant_id) + + row = await self._db.fetch_one(query, params) + if not row: + return None + + result = dict(row) + # Parse JSONB fields + for field in ['exif_data', 'pdf_metadata', 'office_metadata', 'findings', 'timeline_events', 'metadata']: + if result.get(field): + result[field] = self._parse_jsonb(result[field], {}) + return result + + async def get_document_forensic_scans(self, doc_id: str) -> List[Dict[str, Any]]: + """ + Get all forensic scans for a document. + + Args: + doc_id: Document ID + + Returns: + List of scan data dicts + """ + if not self._db: + return [] + + tenant_id = self.get_tenant_id_or_none() + query = "SELECT * FROM arkham_provenance.forensic_scans WHERE doc_id = :doc_id" + params: Dict[str, Any] = {"doc_id": doc_id} + + if tenant_id: + query += " AND tenant_id = :tenant_id" + params["tenant_id"] = str(tenant_id) + + query += " ORDER BY scanned_at DESC" + rows = await self._db.fetch_all(query, params) + + if not rows: + return [] + + results = [] + for row in rows: + result = dict(row) + for field in ['exif_data', 'pdf_metadata', 'office_metadata', 'findings', 'timeline_events', 'metadata']: + if result.get(field): + result[field] = self._parse_jsonb(result[field], {}) + results.append(result) + return results + + async def get_forensic_stats(self) -> Dict[str, Any]: + """ + Get forensic scanning statistics. + + Returns: + Statistics dictionary + """ + if not self._db: + return {} + + tenant_id = self.get_tenant_id_or_none() + tenant_filter = "" + params: Dict[str, Any] = {} + + if tenant_id: + tenant_filter = " WHERE tenant_id = :tenant_id" + params["tenant_id"] = str(tenant_id) + + # Total scans + total_row = await self._db.fetch_one( + f"SELECT COUNT(*) as count FROM arkham_provenance.forensic_scans{tenant_filter}", + params + ) + total_scans = total_row.get("count", 0) if total_row else 0 + + # Scans by status + status_rows = await self._db.fetch_all( + f"""SELECT scan_status, COUNT(*) as count + FROM arkham_provenance.forensic_scans{tenant_filter} + GROUP BY scan_status""", + params + ) + scans_by_status = {row["scan_status"]: row["count"] for row in status_rows} if status_rows else {} + + # Integrity counts + integrity_rows = await self._db.fetch_all( + f"""SELECT integrity_status, COUNT(*) as count + FROM arkham_provenance.forensic_scans{tenant_filter} + GROUP BY integrity_status""", + params + ) + integrity_counts = {row["integrity_status"]: row["count"] for row in integrity_rows} if integrity_rows else {} + + # Documents with findings + findings_row = await self._db.fetch_one( + f"""SELECT COUNT(DISTINCT doc_id) as count + FROM arkham_provenance.forensic_scans + WHERE findings != '[]'{' AND tenant_id = :tenant_id' if tenant_id else ''}""", + params + ) + docs_with_findings = findings_row.get("count", 0) if findings_row else 0 + + # Average confidence + avg_row = await self._db.fetch_one( + f"""SELECT AVG(confidence_score) as avg_confidence + FROM arkham_provenance.forensic_scans{tenant_filter}""", + params + ) + avg_confidence = float(avg_row.get("avg_confidence", 0) or 0) if avg_row else 0.0 + + return { + "total_scans": total_scans, + "scans_by_status": scans_by_status, + "documents_with_findings": docs_with_findings, + "integrity_clean": integrity_counts.get("clean", 0), + "integrity_suspicious": integrity_counts.get("suspicious", 0), + "integrity_tampered": integrity_counts.get("tampered", 0), + "average_confidence": avg_confidence, + } diff --git a/packages/arkham-shard-provenance/pyproject.toml b/packages/arkham-shard-provenance/pyproject.toml index b67cea3..8a858de 100644 --- a/packages/arkham-shard-provenance/pyproject.toml +++ b/packages/arkham-shard-provenance/pyproject.toml @@ -9,6 +9,10 @@ requires-python = ">=3.10" dependencies = [ "arkham-frame>=0.1.0", "pydantic>=2.0.0", + # Metadata forensics dependencies + "Pillow>=10.0.0", # Image EXIF extraction + "pypdf>=4.0.0", # PDF metadata extraction + "python-magic>=0.4.27", # File type detection (requires libmagic system library) ] [project.optional-dependencies] diff --git a/packages/arkham-shard-search/arkham_shard_search/api.py b/packages/arkham-shard-search/arkham_shard_search/api.py index c2e2d58..d0f7346 100644 --- a/packages/arkham-shard-search/arkham_shard_search/api.py +++ b/packages/arkham-shard-search/arkham_shard_search/api.py @@ -32,16 +32,18 @@ _semantic_engine = None _keyword_engine = None _hybrid_engine = None +_regex_engine = None _filter_optimizer = None _event_bus = None -def init_api(semantic_engine, keyword_engine, hybrid_engine, filter_optimizer, event_bus): +def init_api(semantic_engine, keyword_engine, hybrid_engine, regex_engine, filter_optimizer, event_bus): """Initialize API with shard dependencies.""" - global _semantic_engine, _keyword_engine, _hybrid_engine, _filter_optimizer, _event_bus + global _semantic_engine, _keyword_engine, _hybrid_engine, _regex_engine, _filter_optimizer, _event_bus _semantic_engine = semantic_engine _keyword_engine = keyword_engine _hybrid_engine = hybrid_engine + _regex_engine = regex_engine _filter_optimizer = filter_optimizer _event_bus = event_bus @@ -727,3 +729,611 @@ async def submit_ai_feedback(request: Request, body: AIFeedbackRequest): logger.warning(f"AI Feedback not stored (DB unavailable): session={body.session_id}") return AIFeedbackResponse(success=True, feedback_id=feedback_id) + + +# --- Regex Search --- + + +class RegexSearchRequest(BaseModel): + """Request for regex search.""" + pattern: str + flags: list[str] = [] + project_id: str | None = None + document_ids: list[str] | None = None + limit: int = 100 + offset: int = 0 + highlight: bool = True + context_chars: int = 100 + + +class RegexMatchResponse(BaseModel): + """Individual regex match.""" + document_id: str + document_title: str + page_number: int | None + chunk_id: str | None + match_text: str + context: str + start_offset: int + end_offset: int + line_number: int | None + + +class RegexSearchResponse(BaseModel): + """Response for regex search.""" + pattern: str + matches: list[RegexMatchResponse] + total_matches: int + total_chunks_with_matches: int + documents_searched: int + duration_ms: float + error: str | None = None + + +class ValidatePatternRequest(BaseModel): + """Request to validate regex pattern.""" + pattern: str + + +class ValidatePatternResponse(BaseModel): + """Response for pattern validation.""" + valid: bool + error: str | None = None + estimated_performance: str + + +class RegexPresetResponse(BaseModel): + """Regex preset.""" + id: str + name: str + pattern: str + description: str + category: str + flags: list[str] = [] + is_system: bool = True + + +class CreatePresetRequest(BaseModel): + """Request to create custom preset.""" + name: str + pattern: str + description: str = "" + category: str = "custom" + flags: list[str] = [] + + +@router.post("/regex", response_model=RegexSearchResponse) +async def search_regex(request: Request, body: RegexSearchRequest): + """ + Search using regex pattern across document content. + + Supports flags: case_insensitive, multiline, dotall + + The search uses PostgreSQL native regex matching for scalability, + with pattern matching happening in the database. + """ + if not _regex_engine: + raise HTTPException(status_code=503, detail="Regex search not initialized") + + # Validate pattern first + valid, error, perf = _regex_engine.validate_pattern(body.pattern) + if not valid: + raise HTTPException(status_code=400, detail=f"Invalid regex pattern: {error}") + if perf == "dangerous": + raise HTTPException( + status_code=400, + detail="Pattern may cause performance issues (catastrophic backtracking)" + ) + + from .models import RegexSearchQuery + + query = RegexSearchQuery( + pattern=body.pattern, + flags=body.flags, + project_id=body.project_id, + document_ids=body.document_ids, + limit=body.limit, + offset=body.offset, + highlight=body.highlight, + context_chars=body.context_chars, + ) + + try: + result = await _regex_engine.search(query) + except Exception as e: + logger.error(f"Regex search failed: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=f"Search failed: {str(e)}") + + # Emit event (following existing pattern) + if _event_bus: + await _event_bus.emit( + "search.regex.executed", + { + "pattern": body.pattern, + "flags": body.flags, + "matches_count": result.total_matches, + "documents_searched": result.documents_searched, + "duration_ms": result.duration_ms, + }, + source="search-shard", + ) + + return RegexSearchResponse( + pattern=result.pattern, + matches=[ + RegexMatchResponse( + document_id=m.document_id, + document_title=m.document_title, + page_number=m.page_number, + chunk_id=m.chunk_id, + match_text=m.match_text, + context=m.context, + start_offset=m.start_offset, + end_offset=m.end_offset, + line_number=m.line_number, + ) + for m in result.matches + ], + total_matches=result.total_matches, + total_chunks_with_matches=result.total_chunks_with_matches, + documents_searched=result.documents_searched, + duration_ms=result.duration_ms, + error=result.error, + ) + + +@router.post("/regex/validate", response_model=ValidatePatternResponse) +async def validate_regex_pattern(body: ValidatePatternRequest): + """ + Validate a regex pattern and estimate performance. + + Returns validation status, error message (if invalid), + and performance estimate (fast/moderate/slow/dangerous). + """ + if not _regex_engine: + raise HTTPException(status_code=503, detail="Regex search not initialized") + + valid, error, perf = _regex_engine.validate_pattern(body.pattern) + + return ValidatePatternResponse( + valid=valid, + error=error, + estimated_performance=perf, + ) + + +@router.get("/regex/presets", response_model=list[RegexPresetResponse]) +async def get_regex_presets(category: str | None = None): + """ + Get available regex presets. + + Categories: pii, contact, financial, technical, temporal, custom + + Returns both system presets and custom user-defined presets. + """ + if not _regex_engine: + raise HTTPException(status_code=503, detail="Regex search not initialized") + + presets = await _regex_engine.get_presets(category=category) + + return [ + RegexPresetResponse( + id=p["id"], + name=p["name"], + pattern=p["pattern"], + description=p.get("description", ""), + category=p["category"], + flags=p.get("flags", []), + is_system=p.get("is_system", True), + ) + for p in presets + ] + + +@router.post("/regex/presets", response_model=RegexPresetResponse) +async def create_regex_preset(request: Request, body: CreatePresetRequest): + """ + Create a custom regex preset. + + Custom presets are stored in the database and can be + retrieved alongside system presets. + """ + if not _regex_engine: + raise HTTPException(status_code=503, detail="Regex search not initialized") + + # Validate pattern + valid, error, _ = _regex_engine.validate_pattern(body.pattern) + if not valid: + raise HTTPException(status_code=400, detail=f"Invalid pattern: {error}") + + # Get tenant_id if available (following codebase pattern) + shard = get_shard(request) + tenant_id = shard.get_tenant_id_or_none() if hasattr(shard, 'get_tenant_id_or_none') else None + + try: + preset = await _regex_engine.save_custom_preset( + name=body.name, + pattern=body.pattern, + description=body.description, + category=body.category, + flags=body.flags, + tenant_id=str(tenant_id) if tenant_id else None, + ) + except Exception as e: + logger.error(f"Failed to create preset: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=f"Failed to create preset: {str(e)}") + + # Emit event + if _event_bus: + await _event_bus.emit( + "search.regex.preset_created", + {"preset_id": preset["id"], "name": preset["name"]}, + source="search-shard", + ) + + return RegexPresetResponse(**preset) + + +@router.delete("/regex/presets/{preset_id}") +async def delete_regex_preset(request: Request, preset_id: str): + """ + Delete a custom regex preset. + + System presets cannot be deleted. + """ + if not _regex_engine: + raise HTTPException(status_code=503, detail="Regex search not initialized") + + # Get tenant_id if available + shard = get_shard(request) + tenant_id = shard.get_tenant_id_or_none() if hasattr(shard, 'get_tenant_id_or_none') else None + + try: + deleted = await _regex_engine.delete_custom_preset( + preset_id=preset_id, + tenant_id=str(tenant_id) if tenant_id else None, + ) + except Exception as e: + logger.error(f"Failed to delete preset: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=f"Failed to delete preset: {str(e)}") + + if not deleted: + raise HTTPException(status_code=404, detail="Preset not found or is a system preset") + + return {"success": True, "preset_id": preset_id} + + +# --- Pattern Extractions API --- + + +class PatternExtractionResponse(BaseModel): + """Individual pattern extraction.""" + id: str + document_id: str + preset_id: str + preset_name: str | None + category: str | None + match_text: str + context: str | None + page_number: int | None + chunk_id: str | None + start_offset: int | None + end_offset: int | None + line_number: int | None + extracted_at: str | None + + +class ExtractionsListResponse(BaseModel): + """Response for pattern extractions list.""" + extractions: list[PatternExtractionResponse] + total: int + document_id: str | None = None + preset_id: str | None = None + category: str | None = None + + +class ExtractionStatsResponse(BaseModel): + """Response for extraction statistics.""" + total_extractions: int + documents_with_patterns: int + by_category: dict[str, int] + by_preset: dict[str, int] + + +@router.get("/regex/extractions", response_model=ExtractionsListResponse) +async def get_pattern_extractions( + request: Request, + document_id: str | None = None, + preset_id: str | None = None, + category: str | None = None, + limit: int = 100, + offset: int = 0, +): + """ + Get auto-extracted pattern matches. + + These are patterns automatically extracted when documents are parsed. + Filter by document_id, preset_id, or category. + """ + shard = get_shard(request) + + if not shard._db: + raise HTTPException(status_code=503, detail="Database not available") + + query = """ + SELECT id, document_id, preset_id, preset_name, category, match_text, + context, page_number, chunk_id, start_offset, end_offset, + line_number, extracted_at + FROM arkham_search.pattern_extractions + WHERE 1=1 + """ + params: dict = {} + + if document_id: + query += " AND document_id = :document_id" + params["document_id"] = document_id + + if preset_id: + query += " AND preset_id = :preset_id" + params["preset_id"] = preset_id + + if category: + query += " AND category = :category" + params["category"] = category + + # Get total count + count_query = """ + SELECT COUNT(*) as total FROM arkham_search.pattern_extractions WHERE 1=1 + """ + if document_id: + count_query += " AND document_id = :document_id" + if preset_id: + count_query += " AND preset_id = :preset_id" + if category: + count_query += " AND category = :category" + + try: + count_row = await shard._db.fetch_one(count_query, params) + total = count_row["total"] if count_row else 0 + except Exception: + total = 0 + + query += " ORDER BY extracted_at DESC LIMIT :limit OFFSET :offset" + params["limit"] = limit + params["offset"] = offset + + try: + rows = await shard._db.fetch_all(query, params) + except Exception as e: + logger.error(f"Failed to fetch extractions: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=f"Failed to fetch extractions: {str(e)}") + + extractions = [ + PatternExtractionResponse( + id=row["id"], + document_id=row["document_id"], + preset_id=row["preset_id"], + preset_name=row["preset_name"], + category=row["category"], + match_text=row["match_text"], + context=row["context"], + page_number=row["page_number"], + chunk_id=row["chunk_id"], + start_offset=row["start_offset"], + end_offset=row["end_offset"], + line_number=row["line_number"], + extracted_at=str(row["extracted_at"]) if row["extracted_at"] else None, + ) + for row in rows + ] + + return ExtractionsListResponse( + extractions=extractions, + total=total, + document_id=document_id, + preset_id=preset_id, + category=category, + ) + + +@router.get("/regex/extractions/stats", response_model=ExtractionStatsResponse) +async def get_extraction_stats( + request: Request, + project_id: str | None = None, +): + """ + Get statistics on pattern extractions. + + Returns counts by category and preset. + """ + shard = get_shard(request) + + if not shard._db: + raise HTTPException(status_code=503, detail="Database not available") + + try: + # Total extractions + total_row = await shard._db.fetch_one( + "SELECT COUNT(*) as total FROM arkham_search.pattern_extractions" + ) + total = total_row["total"] if total_row else 0 + + # Documents with patterns + doc_row = await shard._db.fetch_one( + "SELECT COUNT(DISTINCT document_id) as cnt FROM arkham_search.pattern_extractions" + ) + docs_with_patterns = doc_row["cnt"] if doc_row else 0 + + # By category + cat_rows = await shard._db.fetch_all( + "SELECT category, COUNT(*) as cnt FROM arkham_search.pattern_extractions GROUP BY category" + ) + by_category = {row["category"]: row["cnt"] for row in cat_rows} if cat_rows else {} + + # By preset + preset_rows = await shard._db.fetch_all( + "SELECT preset_id, COUNT(*) as cnt FROM arkham_search.pattern_extractions GROUP BY preset_id" + ) + by_preset = {row["preset_id"]: row["cnt"] for row in preset_rows} if preset_rows else {} + + except Exception as e: + logger.error(f"Failed to fetch extraction stats: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=f"Failed to fetch stats: {str(e)}") + + return ExtractionStatsResponse( + total_extractions=total, + documents_with_patterns=docs_with_patterns, + by_category=by_category, + by_preset=by_preset, + ) + + +@router.post("/regex/extract/{document_id}") +async def trigger_pattern_extraction( + request: Request, + document_id: str, + preset_ids: list[str] | None = None, +): + """ + Manually trigger pattern extraction for a specific document. + + By default runs all system presets. Optionally specify preset_ids + to run only specific presets. + """ + import re + import uuid + + shard = get_shard(request) + + if not shard._db: + raise HTTPException(status_code=503, detail="Database not available") + + # Check document exists + doc_row = await shard._db.fetch_one( + "SELECT id FROM arkham_frame.documents WHERE id = :doc_id", + {"doc_id": document_id} + ) + if not doc_row: + raise HTTPException(status_code=404, detail="Document not found") + + # Fetch chunks + chunks = await shard._db.fetch_all( + """ + SELECT id, text, chunk_index, page_number + FROM arkham_frame.chunks + WHERE document_id = :doc_id + ORDER BY chunk_index + """, + {"doc_id": document_id} + ) + + if not chunks: + return {"document_id": document_id, "extractions": 0, "message": "No chunks found"} + + # Get presets to use + from .engines.regex import REGEX_PRESETS + + if preset_ids: + presets = [p for p in REGEX_PRESETS if p["id"] in preset_ids] + # Also get custom presets from DB + if shard._db: + custom_query = """ + SELECT id, name, pattern, category FROM arkham_search.regex_presets + WHERE id = ANY(:ids) + """ + custom_rows = await shard._db.fetch_all(custom_query, {"ids": preset_ids}) + for row in custom_rows: + presets.append({ + "id": row["id"], + "name": row["name"], + "pattern": row["pattern"], + "category": row["category"], + }) + else: + presets = list(REGEX_PRESETS) + + # Clear existing extractions for this document (re-extract) + await shard._db.execute( + "DELETE FROM arkham_search.pattern_extractions WHERE document_id = :doc_id", + {"doc_id": document_id} + ) + + # Extract patterns + extractions = [] + for preset in presets: + preset_id = preset["id"] + preset_name = preset["name"] + category = preset["category"] + pattern = preset["pattern"] + + try: + compiled = re.compile(pattern, re.IGNORECASE) + except re.error: + continue + + for chunk in chunks: + text = chunk["text"] or "" + chunk_id = chunk["id"] + page_number = chunk["page_number"] + + for match in compiled.finditer(text): + match_text = match.group(0) + match_start = match.start() + match_end = match.end() + + ctx_start = max(0, match_start - 50) + ctx_end = min(len(text), match_end + 50) + context = text[ctx_start:ctx_end] + if ctx_start > 0: + context = "..." + context + if ctx_end < len(text): + context = context + "..." + + line_number = text[:match_start].count('\n') + 1 + + extractions.append({ + "id": str(uuid.uuid4())[:12], + "document_id": document_id, + "preset_id": preset_id, + "preset_name": preset_name, + "category": category, + "match_text": match_text, + "context": context, + "page_number": page_number, + "chunk_id": chunk_id, + "start_offset": match_start, + "end_offset": match_end, + "line_number": line_number, + }) + + # Store extractions + for ext in extractions: + await shard._db.execute( + """ + INSERT INTO arkham_search.pattern_extractions + (id, document_id, preset_id, preset_name, category, match_text, + context, page_number, chunk_id, start_offset, end_offset, line_number) + VALUES (:id, :document_id, :preset_id, :preset_name, :category, :match_text, + :context, :page_number, :chunk_id, :start_offset, :end_offset, :line_number) + ON CONFLICT (id) DO NOTHING + """, + ext + ) + + # Emit event + if _event_bus: + await _event_bus.emit( + "search.patterns.extracted", + { + "document_id": document_id, + "total_matches": len(extractions), + "manual_trigger": True, + }, + source="search-shard", + ) + + return { + "document_id": document_id, + "extractions": len(extractions), + "presets_used": [p["id"] for p in presets], + } diff --git a/packages/arkham-shard-search/arkham_shard_search/engines/__init__.py b/packages/arkham-shard-search/arkham_shard_search/engines/__init__.py index 182440d..7b4de8e 100644 --- a/packages/arkham-shard-search/arkham_shard_search/engines/__init__.py +++ b/packages/arkham-shard-search/arkham_shard_search/engines/__init__.py @@ -3,5 +3,6 @@ from .semantic import SemanticSearchEngine from .keyword import KeywordSearchEngine from .hybrid import HybridSearchEngine +from .regex import RegexSearchEngine -__all__ = ["SemanticSearchEngine", "KeywordSearchEngine", "HybridSearchEngine"] +__all__ = ["SemanticSearchEngine", "KeywordSearchEngine", "HybridSearchEngine", "RegexSearchEngine"] diff --git a/packages/arkham-shard-search/arkham_shard_search/engines/regex.py b/packages/arkham-shard-search/arkham_shard_search/engines/regex.py new file mode 100644 index 0000000..a0fcdf3 --- /dev/null +++ b/packages/arkham-shard-search/arkham_shard_search/engines/regex.py @@ -0,0 +1,458 @@ +"""Regex search engine for pattern matching across documents.""" + +import re +import logging +import time +from typing import Any + +from ..models import RegexMatch, RegexSearchQuery, RegexSearchResult + +logger = logging.getLogger(__name__) + + +# Built-in presets (following codebase constant pattern) +REGEX_PRESETS = [ + { + "id": "email", + "name": "Email Addresses", + "pattern": r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", + "description": "Match email addresses", + "category": "contact", + }, + { + "id": "phone_us", + "name": "US Phone Numbers", + "pattern": r"\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}", + "description": "Match US phone number formats", + "category": "contact", + }, + { + "id": "ssn", + "name": "Social Security Numbers", + "pattern": r"\d{3}-\d{2}-\d{4}", + "description": "Match SSN format (XXX-XX-XXXX)", + "category": "pii", + }, + { + "id": "credit_card", + "name": "Credit Card Numbers", + "pattern": r"\d{4}[-\s]?\d{4}[-\s]?\d{4}[-\s]?\d{4}", + "description": "Match credit card number format", + "category": "financial", + }, + { + "id": "ip_address", + "name": "IP Addresses", + "pattern": r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", + "description": "Match IPv4 addresses", + "category": "technical", + }, + { + "id": "date_mdy", + "name": "Dates (MM/DD/YYYY)", + "pattern": r"\d{1,2}/\d{1,2}/\d{2,4}", + "description": "Match dates in MM/DD/YYYY format", + "category": "temporal", + }, + { + "id": "url", + "name": "URLs", + "pattern": r"https?://[^\s]+", + "description": "Match HTTP/HTTPS URLs", + "category": "technical", + }, + { + "id": "money_usd", + "name": "USD Amounts", + "pattern": r"\$[\d,]+\.?\d*", + "description": "Match US dollar amounts", + "category": "financial", + }, +] + + +class RegexSearchEngine: + """ + Regex search engine that searches across document content. + + Follows the same pattern as SemanticSearchEngine and KeywordSearchEngine. + + IMPORTANT: Uses PostgreSQL native regex (~, ~*) for scalability. + Pattern matching happens in the database, not in Python memory. + This prevents memory blowup on large corpora. + """ + + def __init__(self, database_service, documents_service=None, config=None): + """ + Initialize regex search engine. + + Args: + database_service: Frame database service (required) + documents_service: Frame documents service (optional) + config: Configuration dict (optional) + """ + self._db = database_service + self._documents_service = documents_service + self._config = config or {} + self._max_results = self._config.get("max_results", 1000) + self._context_chars = self._config.get("context_chars", 100) + self._timeout_ms = self._config.get("timeout_ms", 30000) + + def validate_pattern(self, pattern: str) -> tuple[bool, str | None, str]: + """ + Validate regex pattern and estimate performance. + + Args: + pattern: Regex pattern to validate + + Returns: + Tuple of (is_valid, error_message, performance_estimate) + """ + try: + re.compile(pattern) + except re.error as e: + return False, str(e), "invalid" + + # Check for dangerous patterns (catastrophic backtracking) + dangerous_patterns = [ + r'(.+)+', # Nested quantifiers + r'(.*)*', # Nested stars + r'(a|a)+', # Exponential alternatives + r'([^"]*)*', # Nested negated character class with star + ] + + for dp in dangerous_patterns: + if dp in pattern: + return True, None, "dangerous" + + # Estimate performance based on pattern complexity + if len(pattern) < 10 and not any(c in pattern for c in '*+?{}'): + return True, None, "fast" + elif len(pattern) < 50: + return True, None, "moderate" + else: + return True, None, "slow" + + async def search(self, query: RegexSearchQuery) -> RegexSearchResult: + """ + Search for regex pattern across document content. + + IMPORTANT: Uses PostgreSQL native regex (~, ~*) for scalability. + Pattern matching happens in the database, not in Python memory. + This prevents memory blowup on large corpora. + + Args: + query: RegexSearchQuery with pattern and filters + + Returns: + RegexSearchResult with matches + """ + start_time = time.time() + + # Validate pattern syntax using Python regex (for better error messages) + try: + re.compile(query.pattern) + except re.error as e: + logger.error(f"Invalid regex pattern: {e}") + return RegexSearchResult( + pattern=query.pattern, + matches=[], + total_matches=0, + documents_searched=0, + duration_ms=(time.time() - start_time) * 1000, + error=f"Invalid regex: {e}", + ) + + # Build PostgreSQL regex operator based on flags + # ~ = case-sensitive match + # ~* = case-insensitive match + case_insensitive = any(f in ("case_insensitive", "i") for f in query.flags) + regex_op = "~*" if case_insensitive else "~" + + # PostgreSQL regex uses different syntax for some features: + # - DOTALL (s flag) not directly supported, but newlines match . in bracket expressions + # - MULTILINE (m flag) is default behavior in PostgreSQL + pg_pattern = query.pattern + + # First, get total count (for pagination info) - uses COUNT with regex + count_sql = f""" + SELECT COUNT(DISTINCT c.id) as match_count, + COUNT(DISTINCT c.document_id) as doc_count + FROM arkham_frame.chunks c + JOIN arkham_frame.documents d ON c.document_id = d.id + WHERE c.text {regex_op} :pattern + """ + params: dict[str, Any] = {"pattern": pg_pattern} + + if query.project_id: + count_sql += " AND d.project_id = :project_id" + params["project_id"] = query.project_id + + if query.document_ids: + count_sql += " AND d.id = ANY(:document_ids)" + params["document_ids"] = query.document_ids + + try: + count_row = await self._db.fetch_one(count_sql, params) + total_chunks_with_matches = count_row["match_count"] if count_row else 0 + documents_searched = count_row["doc_count"] if count_row else 0 + except Exception as e: + logger.error(f"Regex count query failed: {e}", exc_info=True) + return RegexSearchResult( + pattern=query.pattern, + matches=[], + total_matches=0, + documents_searched=0, + duration_ms=(time.time() - start_time) * 1000, + error=f"Database error: {e}", + ) + + # Now fetch actual matches with pagination + # Use subquery for matching chunks, then extract match positions in Python + # (PostgreSQL regexp_matches with 'g' flag returns multiple rows per match, + # which can be complex to handle in SQL - we'll extract in Python) + search_sql = f""" + SELECT c.id as chunk_id, c.document_id, c.text, c.chunk_index, + d.filename as document_title, c.page_number + FROM arkham_frame.chunks c + JOIN arkham_frame.documents d ON c.document_id = d.id + WHERE c.text {regex_op} :pattern + """ + + if query.project_id: + search_sql += " AND d.project_id = :project_id" + + if query.document_ids: + search_sql += " AND d.id = ANY(:document_ids)" + + search_sql += """ + ORDER BY d.id, c.chunk_index + LIMIT :chunk_limit + """ + + params["chunk_limit"] = min(1000, self._max_results) # Cap chunks to prevent memory issues + + matches = [] + try: + rows = await self._db.fetch_all(search_sql, params) + + # Build Python regex for match extraction with flags + py_flags = 0 + if case_insensitive: + py_flags |= re.IGNORECASE + if "multiline" in query.flags: + py_flags |= re.MULTILINE + if "dotall" in query.flags: + py_flags |= re.DOTALL + + compiled_pattern = re.compile(query.pattern, py_flags) + + for row in rows: + text = row["text"] or "" + document_id = row["document_id"] + document_title = row["document_title"] or "" + page_number = row["page_number"] + chunk_id = row["chunk_id"] + + # Find all matches in this chunk's text + for match in compiled_pattern.finditer(text): + match_text = match.group(0) + match_start = match.start() + match_end = match.end() + + # Extract context around match + context_chars = query.context_chars + ctx_start = max(0, match_start - context_chars) + ctx_end = min(len(text), match_end + context_chars) + context = text[ctx_start:ctx_end] + + # Add ellipsis if truncated + if ctx_start > 0: + context = "..." + context + if ctx_end < len(text): + context = context + "..." + + # Calculate line number + line_number = text[:match_start].count('\n') + 1 + + matches.append(RegexMatch( + document_id=document_id, + document_title=document_title, + page_number=page_number, + chunk_id=chunk_id, + match_text=match_text, + context=context, + start_offset=match_start, + end_offset=match_end, + line_number=line_number, + )) + + if len(matches) >= self._max_results: + break + + if len(matches) >= self._max_results: + break + + except Exception as e: + logger.error(f"Regex search failed: {e}", exc_info=True) + return RegexSearchResult( + pattern=query.pattern, + matches=[], + total_matches=0, + documents_searched=0, + duration_ms=(time.time() - start_time) * 1000, + error=f"Search failed: {e}", + ) + + # Apply pagination to results + paginated_matches = matches[query.offset:query.offset + query.limit] + + duration_ms = (time.time() - start_time) * 1000 + + logger.info( + f"Regex search '{query.pattern}' found {len(matches)} matches " + f"in {total_chunks_with_matches} chunks across {documents_searched} documents " + f"({duration_ms:.1f}ms)" + ) + + return RegexSearchResult( + pattern=query.pattern, + matches=paginated_matches, + total_matches=len(matches), # Total from this batch + total_chunks_with_matches=total_chunks_with_matches, # True total + documents_searched=documents_searched, + duration_ms=duration_ms, + ) + + async def get_presets(self, category: str | None = None) -> list[dict]: + """ + Get available regex presets (system + custom from database). + + Args: + category: Optional category filter + + Returns: + List of preset dictionaries + """ + # Start with system presets + presets = list(REGEX_PRESETS) + + # Add custom presets from database + if self._db: + try: + query = """ + SELECT id, name, pattern, description, category, flags, is_system + FROM arkham_search.regex_presets + WHERE 1=1 + """ + params: dict[str, Any] = {} + + if category: + query += " AND category = :category" + params["category"] = category + + query += " ORDER BY name" + + rows = await self._db.fetch_all(query, params) + for row in rows: + import json + presets.append({ + "id": row["id"], + "name": row["name"], + "pattern": row["pattern"], + "description": row.get("description", ""), + "category": row["category"], + "flags": json.loads(row["flags"]) if isinstance(row["flags"], str) else (row["flags"] or []), + "is_system": row.get("is_system", False), + }) + except Exception as e: + logger.warning(f"Failed to fetch custom presets: {e}") + + # Filter system presets by category if specified + if category: + presets = [p for p in presets if p["category"] == category] + + return presets + + async def save_custom_preset( + self, + name: str, + pattern: str, + description: str = "", + category: str = "custom", + flags: list[str] | None = None, + tenant_id: str | None = None, + ) -> dict: + """ + Save a custom regex preset to the database. + + Args: + name: Preset name + pattern: Regex pattern + description: Optional description + category: Category (default: custom) + flags: Optional regex flags + tenant_id: Optional tenant ID for multi-tenancy + + Returns: + Created preset dictionary + """ + import uuid + import json + + preset_id = str(uuid.uuid4())[:8] + + await self._db.execute( + """ + INSERT INTO arkham_search.regex_presets + (id, tenant_id, name, pattern, description, category, flags, is_system, created_at) + VALUES (:id, :tenant_id, :name, :pattern, :description, :category, :flags, FALSE, CURRENT_TIMESTAMP) + """, + { + "id": preset_id, + "tenant_id": tenant_id, + "name": name, + "pattern": pattern, + "description": description, + "category": category, + "flags": json.dumps(flags or []), + } + ) + + return { + "id": preset_id, + "name": name, + "pattern": pattern, + "description": description, + "category": category, + "flags": flags or [], + "is_system": False, + } + + async def delete_custom_preset(self, preset_id: str, tenant_id: str | None = None) -> bool: + """ + Delete a custom regex preset. + + Args: + preset_id: Preset ID to delete + tenant_id: Optional tenant ID for multi-tenancy + + Returns: + True if deleted successfully + """ + try: + query = """ + DELETE FROM arkham_search.regex_presets + WHERE id = :id AND is_system = FALSE + """ + params: dict[str, Any] = {"id": preset_id} + + if tenant_id: + query += " AND tenant_id = :tenant_id" + params["tenant_id"] = tenant_id + + await self._db.execute(query, params) + return True + except Exception as e: + logger.error(f"Failed to delete preset: {e}") + return False diff --git a/packages/arkham-shard-search/arkham_shard_search/models.py b/packages/arkham-shard-search/arkham_shard_search/models.py index 518670a..049e1cc 100644 --- a/packages/arkham-shard-search/arkham_shard_search/models.py +++ b/packages/arkham-shard-search/arkham_shard_search/models.py @@ -11,6 +11,14 @@ class SearchMode(Enum): HYBRID = "hybrid" SEMANTIC = "semantic" KEYWORD = "keyword" + REGEX = "regex" + + +class RegexFlag(Enum): + """Regex flags for pattern matching.""" + CASE_INSENSITIVE = "case_insensitive" + MULTILINE = "multiline" + DOTALL = "dotall" class SortBy(Enum): @@ -119,3 +127,57 @@ class SimilarityRequest: limit: int = 10 min_similarity: float = 0.5 filters: SearchFilters | None = None + + +# --- Regex Search Models --- + + +@dataclass +class RegexMatch: + """Individual regex match result.""" + document_id: str + document_title: str + page_number: int | None + chunk_id: str | None + match_text: str + context: str + start_offset: int + end_offset: int + line_number: int | None = None + + +@dataclass +class RegexSearchQuery: + """Regex search query parameters.""" + pattern: str + flags: list[str] = field(default_factory=list) + project_id: str | None = None + document_ids: list[str] | None = None + limit: int = 100 + offset: int = 0 + highlight: bool = True + context_chars: int = 100 + + +@dataclass +class RegexSearchResult: + """Regex search result container.""" + pattern: str + matches: list[RegexMatch] = field(default_factory=list) + total_matches: int = 0 + total_chunks_with_matches: int = 0 + documents_searched: int = 0 + duration_ms: float = 0.0 + error: str | None = None + + +@dataclass +class RegexPreset: + """Predefined regex pattern.""" + id: str + name: str + pattern: str + description: str + category: str # pii, contact, financial, technical, custom + flags: list[str] = field(default_factory=list) + is_system: bool = True diff --git a/packages/arkham-shard-search/arkham_shard_search/shard.py b/packages/arkham-shard-search/arkham_shard_search/shard.py index 8b80f77..8997d60 100644 --- a/packages/arkham-shard-search/arkham_shard_search/shard.py +++ b/packages/arkham-shard-search/arkham_shard_search/shard.py @@ -5,7 +5,7 @@ from arkham_frame.shard_interface import ArkhamShard from .api import init_api, router -from .engines import SemanticSearchEngine, KeywordSearchEngine, HybridSearchEngine +from .engines import SemanticSearchEngine, KeywordSearchEngine, HybridSearchEngine, RegexSearchEngine from .filters import FilterOptimizer logger = logging.getLogger(__name__) @@ -33,6 +33,7 @@ def __init__(self): self.semantic_engine = None self.keyword_engine = None self.hybrid_engine = None + self.regex_engine = None self.filter_optimizer = None self._db = None @@ -95,6 +96,14 @@ async def initialize(self, frame) -> None: if database_service: self.filter_optimizer = FilterOptimizer(database_service) + # Initialize regex search engine + if database_service: + self.regex_engine = RegexSearchEngine( + database_service=database_service, + documents_service=documents_service, + ) + logger.info("Regex search engine initialized") + # Create search schema and tables if database_service: await self._create_schema(database_service) @@ -104,6 +113,7 @@ async def initialize(self, frame) -> None: semantic_engine=self.semantic_engine, keyword_engine=self.keyword_engine, hybrid_engine=self.hybrid_engine, + regex_engine=self.regex_engine, filter_optimizer=self.filter_optimizer, event_bus=event_bus, ) @@ -112,6 +122,8 @@ async def initialize(self, frame) -> None: if event_bus: await event_bus.subscribe("documents.indexed", self._on_document_indexed) await event_bus.subscribe("documents.deleted", self._on_document_deleted) + # Subscribe to parse completion for auto-extraction of regex patterns + await event_bus.subscribe("parse.document.completed", self._on_parse_completed) # Register self in app state for API access if hasattr(frame, "app") and frame.app: @@ -129,10 +141,12 @@ async def shutdown(self) -> None: if event_bus: await event_bus.unsubscribe("documents.indexed", self._on_document_indexed) await event_bus.unsubscribe("documents.deleted", self._on_document_deleted) + await event_bus.unsubscribe("parse.document.completed", self._on_parse_completed) self.semantic_engine = None self.keyword_engine = None self.hybrid_engine = None + self.regex_engine = None self.filter_optimizer = None logger.info("Search Shard shutdown complete") @@ -163,7 +177,7 @@ async def _create_schema(self, db) -> None: ) """) - # Create indexes + # Create indexes for AI feedback await db.execute(""" CREATE INDEX IF NOT EXISTS idx_ai_feedback_session ON arkham_search.ai_feedback(session_id) @@ -177,6 +191,90 @@ async def _create_schema(self, db) -> None: ON arkham_search.ai_feedback(created_at DESC) """) + # Regex presets table for custom patterns + await db.execute(""" + CREATE TABLE IF NOT EXISTS arkham_search.regex_presets ( + id TEXT PRIMARY KEY, + tenant_id UUID, + name TEXT NOT NULL, + pattern TEXT NOT NULL, + description TEXT, + category TEXT DEFAULT 'custom', + flags JSONB DEFAULT '[]', + is_system BOOLEAN DEFAULT FALSE, + created_by TEXT, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + usage_count INTEGER DEFAULT 0 + ) + """) + + # Regex search history for analytics + await db.execute(""" + CREATE TABLE IF NOT EXISTS arkham_search.regex_history ( + id TEXT PRIMARY KEY, + tenant_id UUID, + pattern TEXT NOT NULL, + flags JSONB DEFAULT '[]', + matches_found INTEGER DEFAULT 0, + documents_searched INTEGER DEFAULT 0, + search_time_ms REAL, + searched_by TEXT, + searched_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + """) + + # Indexes for regex tables + await db.execute(""" + CREATE INDEX IF NOT EXISTS idx_regex_presets_tenant + ON arkham_search.regex_presets(tenant_id) + """) + await db.execute(""" + CREATE INDEX IF NOT EXISTS idx_regex_presets_category + ON arkham_search.regex_presets(category) + """) + await db.execute(""" + CREATE INDEX IF NOT EXISTS idx_regex_history_tenant + ON arkham_search.regex_history(tenant_id) + """) + await db.execute(""" + CREATE INDEX IF NOT EXISTS idx_regex_history_searched_at + ON arkham_search.regex_history(searched_at DESC) + """) + + # Auto-extracted pattern matches table + await db.execute(""" + CREATE TABLE IF NOT EXISTS arkham_search.pattern_extractions ( + id TEXT PRIMARY KEY, + document_id TEXT NOT NULL, + preset_id TEXT NOT NULL, + preset_name TEXT, + category TEXT, + match_text TEXT NOT NULL, + context TEXT, + page_number INTEGER, + chunk_id TEXT, + start_offset INTEGER, + end_offset INTEGER, + line_number INTEGER, + extracted_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + tenant_id UUID + ) + """) + + # Indexes for pattern extractions + await db.execute(""" + CREATE INDEX IF NOT EXISTS idx_pattern_extractions_doc + ON arkham_search.pattern_extractions(document_id) + """) + await db.execute(""" + CREATE INDEX IF NOT EXISTS idx_pattern_extractions_preset + ON arkham_search.pattern_extractions(preset_id) + """) + await db.execute(""" + CREATE INDEX IF NOT EXISTS idx_pattern_extractions_category + ON arkham_search.pattern_extractions(category) + """) + logger.info("Search schema and tables created") except Exception as e: logger.warning(f"Failed to create search schema: {e}") @@ -252,10 +350,142 @@ async def _on_document_deleted(self, event: dict) -> None: - Remove from search indexes - Clean up cache entries """ - doc_id = event.get("doc_id") + doc_id = event.get("doc_id") or event.get("document_id") logger.debug(f"Document deleted: {doc_id}") - # TODO: Remove from any search caches + # Clean up pattern extractions for deleted document + if self._db and doc_id: + try: + await self._db.execute( + "DELETE FROM arkham_search.pattern_extractions WHERE document_id = :doc_id", + {"doc_id": doc_id} + ) + except Exception as e: + logger.warning(f"Failed to delete pattern extractions for {doc_id}: {e}") + + async def _on_parse_completed(self, event: dict) -> None: + """ + Handle parse completion event - auto-extract regex patterns from document. + + Runs all system presets against the newly parsed document and stores + the matches for quick retrieval. + """ + import re + import uuid + + # Handle both wrapped and unwrapped event payloads + payload = event.get("payload", event) + doc_id = payload.get("document_id") + + if not doc_id: + logger.debug("No document_id in parse.document.completed event") + return + + if not self._db or not self.regex_engine: + logger.debug("Database or regex engine not available for pattern extraction") + return + + logger.info(f"Auto-extracting regex patterns for document {doc_id}") + + try: + # Fetch all chunks for this document + chunks = await self._db.fetch_all( + """ + SELECT id, text, chunk_index, page_number + FROM arkham_frame.chunks + WHERE document_id = :doc_id + ORDER BY chunk_index + """, + {"doc_id": doc_id} + ) + + if not chunks: + logger.debug(f"No chunks found for document {doc_id}") + return + + # Get all system presets + from .engines.regex import REGEX_PRESETS + + extractions = [] + for preset in REGEX_PRESETS: + preset_id = preset["id"] + preset_name = preset["name"] + category = preset["category"] + pattern = preset["pattern"] + + try: + compiled = re.compile(pattern, re.IGNORECASE) + except re.error: + continue + + for chunk in chunks: + text = chunk["text"] or "" + chunk_id = chunk["id"] + page_number = chunk["page_number"] + + for match in compiled.finditer(text): + match_text = match.group(0) + match_start = match.start() + match_end = match.end() + + # Extract context + ctx_start = max(0, match_start - 50) + ctx_end = min(len(text), match_end + 50) + context = text[ctx_start:ctx_end] + if ctx_start > 0: + context = "..." + context + if ctx_end < len(text): + context = context + "..." + + line_number = text[:match_start].count('\n') + 1 + + extractions.append({ + "id": str(uuid.uuid4())[:12], + "document_id": doc_id, + "preset_id": preset_id, + "preset_name": preset_name, + "category": category, + "match_text": match_text, + "context": context, + "page_number": page_number, + "chunk_id": chunk_id, + "start_offset": match_start, + "end_offset": match_end, + "line_number": line_number, + }) + + # Store extractions in batch + if extractions: + for ext in extractions: + await self._db.execute( + """ + INSERT INTO arkham_search.pattern_extractions + (id, document_id, preset_id, preset_name, category, match_text, + context, page_number, chunk_id, start_offset, end_offset, line_number) + VALUES (:id, :document_id, :preset_id, :preset_name, :category, :match_text, + :context, :page_number, :chunk_id, :start_offset, :end_offset, :line_number) + ON CONFLICT (id) DO NOTHING + """, + ext + ) + + logger.info(f"Auto-extracted {len(extractions)} pattern matches from document {doc_id}") + + # Emit event for pattern extraction completion + event_bus = self.frame.get_service("events") if self.frame else None + if event_bus: + await event_bus.emit( + "search.patterns.extracted", + { + "document_id": doc_id, + "total_matches": len(extractions), + "categories": list(set(e["category"] for e in extractions)), + }, + source="search-shard", + ) + + except Exception as e: + logger.error(f"Failed to extract patterns for document {doc_id}: {e}", exc_info=True) # --- Public API for other shards --- diff --git a/packages/arkham-shard-shell/src/pages/anomalies/AnomaliesPage.css b/packages/arkham-shard-shell/src/pages/anomalies/AnomaliesPage.css index 1dde57b..7346aff 100644 --- a/packages/arkham-shard-shell/src/pages/anomalies/AnomaliesPage.css +++ b/packages/arkham-shard-shell/src/pages/anomalies/AnomaliesPage.css @@ -1248,3 +1248,646 @@ .dialog { animation: slideUp 0.2s ease-out; } + +/* ============================================ + Tab Navigation + ============================================ */ + +.anomalies-tabs { + display: flex; + gap: 0; + border-bottom: 1px solid var(--arkham-border); + margin-bottom: 1.5rem; +} + +.anomalies-tab { + display: flex; + align-items: center; + gap: 0.5rem; + padding: 0.75rem 1.25rem; + font-size: 0.875rem; + font-weight: 500; + color: var(--arkham-text-secondary); + background: transparent; + border: none; + border-bottom: 2px solid transparent; + cursor: pointer; + transition: all 0.15s ease; +} + +.anomalies-tab:hover { + color: var(--arkham-text-primary); + background: var(--arkham-bg-tertiary); +} + +.anomalies-tab.active { + color: var(--arkham-accent-primary); + border-bottom-color: var(--arkham-accent-primary); +} + +/* ============================================ + Hidden Content Tab + ============================================ */ + +.hidden-content-tab { + display: flex; + flex-direction: column; + gap: 1.5rem; +} + +/* Stats Grid */ +.hc-stats-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); + gap: 1rem; +} + +.hc-stat-card { + display: flex; + flex-direction: column; + align-items: center; + gap: 0.5rem; + padding: 1.25rem; + background: var(--arkham-bg-secondary); + border: 1px solid var(--arkham-border); + border-radius: var(--arkham-radius-md); + text-align: center; +} + +.hc-stat-card svg { + color: var(--arkham-accent-primary); + opacity: 0.8; +} + +.hc-stat-value { + font-size: 1.75rem; + font-weight: 700; + color: var(--arkham-text-primary); +} + +.hc-stat-label { + font-size: 0.75rem; + color: var(--arkham-text-secondary); + text-transform: uppercase; + letter-spacing: 0.05em; +} + +/* Scan Controls */ +.hc-scan-controls { + display: flex; + gap: 1rem; + padding: 1.25rem; + background: var(--arkham-bg-secondary); + border: 1px solid var(--arkham-border); + border-radius: var(--arkham-radius-md); + flex-wrap: wrap; + align-items: flex-end; +} + +.hc-scan-form { + display: flex; + flex-direction: column; + gap: 0.5rem; + flex: 1; + min-width: 250px; +} + +.hc-scan-form label { + font-size: 0.875rem; + font-weight: 500; + color: var(--arkham-text-secondary); +} + +.hc-scan-form select { + padding: 0.625rem 0.875rem; + border: 1px solid var(--arkham-border); + border-radius: var(--arkham-radius-sm); + background: var(--arkham-bg-primary); + color: var(--arkham-text-primary); + font-size: 0.875rem; +} + +.hc-scan-form select:focus { + outline: none; + border-color: var(--arkham-accent-primary); +} + +.hc-scan-actions { + display: flex; + gap: 0.5rem; +} + +/* Scan Results */ +.hc-scan-results { + padding: 1.5rem; + background: var(--arkham-bg-secondary); + border: 1px solid var(--arkham-border); + border-radius: var(--arkham-radius-md); +} + +.hc-scan-results h3 { + display: flex; + align-items: center; + gap: 0.5rem; + margin: 0 0 1.25rem 0; + font-size: 1.125rem; + font-weight: 600; + color: var(--arkham-text-primary); +} + +.hc-scan-results h4 { + display: flex; + align-items: center; + gap: 0.5rem; + margin: 0 0 0.75rem 0; + font-size: 0.9375rem; + font-weight: 600; + color: var(--arkham-text-primary); +} + +/* Result Summary */ +.hc-result-summary { + display: flex; + gap: 1.5rem; + padding: 1rem; + background: var(--arkham-bg-tertiary); + border-radius: var(--arkham-radius-sm); + margin-bottom: 1.5rem; + flex-wrap: wrap; + align-items: center; +} + +.hc-confidence-badge { + display: flex; + flex-direction: column; + align-items: center; + padding: 1rem 1.5rem; + border-radius: var(--arkham-radius-md); + min-width: 120px; +} + +.hc-confidence-badge .confidence-value { + font-size: 2rem; + font-weight: 700; +} + +.hc-confidence-badge .confidence-label { + font-size: 0.6875rem; + text-transform: uppercase; + letter-spacing: 0.05em; + opacity: 0.9; +} + +.hc-confidence-badge.low { + background: rgba(74, 222, 128, 0.15); + color: var(--arkham-success); +} + +.hc-confidence-badge.medium { + background: rgba(251, 191, 36, 0.15); + color: var(--arkham-warning); +} + +.hc-confidence-badge.high { + background: rgba(251, 146, 60, 0.15); + color: #fb923c; +} + +.hc-confidence-badge.critical { + background: rgba(248, 113, 113, 0.15); + color: var(--arkham-error); +} + +.hc-summary-stats { + display: flex; + flex-direction: column; + gap: 0.5rem; + flex: 1; +} + +.hc-summary-stats .summary-item { + display: flex; + justify-content: space-between; + padding: 0.375rem 0; + border-bottom: 1px solid var(--arkham-border); +} + +.hc-summary-stats .summary-item:last-child { + border-bottom: none; +} + +.hc-summary-stats .label { + font-size: 0.875rem; + color: var(--arkham-text-secondary); +} + +.hc-summary-stats .value { + font-size: 0.875rem; + font-weight: 600; + color: var(--arkham-text-primary); +} + +.hc-summary-stats .value.high { + color: var(--arkham-error); +} + +.hc-summary-stats .value.medium { + color: var(--arkham-warning); +} + +.hc-summary-stats .value.success { + color: var(--arkham-success); +} + +.hc-summary-stats .value.error { + color: var(--arkham-error); +} + +/* Findings */ +.hc-findings { + margin-bottom: 1.5rem; + padding: 1rem; + background: rgba(248, 113, 113, 0.05); + border: 1px solid rgba(248, 113, 113, 0.2); + border-radius: var(--arkham-radius-sm); +} + +.hc-findings ul { + margin: 0; + padding-left: 1.25rem; +} + +.hc-findings li { + font-size: 0.875rem; + color: var(--arkham-text-primary); + margin-bottom: 0.375rem; + line-height: 1.5; +} + +/* Entropy Regions */ +.hc-entropy-regions { + margin-bottom: 1.5rem; +} + +.entropy-chart { + display: flex; + flex-direction: column; + gap: 0.75rem; +} + +.entropy-region { + display: flex; + flex-direction: column; + gap: 0.25rem; +} + +.entropy-region .region-label { + display: flex; + align-items: center; + gap: 0.5rem; + font-size: 0.8125rem; + font-weight: 500; + color: var(--arkham-text-primary); +} + +.entropy-region .region-range { + font-size: 0.75rem; + color: var(--arkham-text-muted); + font-weight: 400; +} + +.region-bar-container { + display: flex; + align-items: center; + gap: 0.75rem; + height: 24px; + background: var(--arkham-bg-tertiary); + border-radius: 4px; + overflow: hidden; + position: relative; +} + +.region-bar { + height: 100%; + border-radius: 4px; + transition: width 0.3s ease; +} + +.region-bar.normal { + background: var(--arkham-success); +} + +.region-bar.medium { + background: var(--arkham-warning); +} + +.region-bar.high { + background: #fb923c; +} + +.region-bar.anomalous { + background: var(--arkham-error); +} + +.region-value { + position: absolute; + right: 0.5rem; + font-size: 0.75rem; + font-weight: 600; + color: var(--arkham-text-primary); +} + +.region-desc { + font-size: 0.75rem; + color: var(--arkham-text-secondary); + padding-left: 0.25rem; +} + +.region-desc.warning { + color: var(--arkham-error); + font-weight: 500; +} + +/* Indicators */ +.hc-indicators { + margin-bottom: 1.5rem; +} + +.indicators-list { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(280px, 1fr)); + gap: 0.75rem; +} + +.indicator-card { + display: flex; + gap: 0.75rem; + padding: 0.875rem; + background: var(--arkham-bg-tertiary); + border: 1px solid var(--arkham-border); + border-radius: var(--arkham-radius-sm); +} + +.indicator-card.high-conf { + border-color: var(--arkham-error); + background: rgba(248, 113, 113, 0.05); +} + +.indicator-icon { + display: flex; + align-items: center; + justify-content: center; + width: 40px; + height: 40px; + background: var(--arkham-bg-secondary); + border-radius: var(--arkham-radius-sm); + color: var(--arkham-accent-primary); + flex-shrink: 0; +} + +.indicator-content { + display: flex; + flex-direction: column; + gap: 0.25rem; + min-width: 0; +} + +.indicator-type { + font-size: 0.875rem; + font-weight: 600; + color: var(--arkham-text-primary); + text-transform: capitalize; +} + +.indicator-location { + font-size: 0.75rem; + color: var(--arkham-text-secondary); +} + +.indicator-confidence { + font-size: 0.75rem; + font-weight: 500; + color: var(--arkham-accent-primary); +} + +/* LSB Analysis */ +.hc-lsb-analysis { + margin-bottom: 1.5rem; +} + +.lsb-details { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(150px, 1fr)); + gap: 0.75rem; + padding: 1rem; + background: var(--arkham-bg-tertiary); + border-radius: var(--arkham-radius-sm); +} + +.lsb-item { + display: flex; + flex-direction: column; + gap: 0.25rem; +} + +.lsb-item .label { + font-size: 0.75rem; + color: var(--arkham-text-secondary); +} + +.lsb-item .value { + font-size: 0.875rem; + font-weight: 600; + color: var(--arkham-text-primary); +} + +.lsb-item .value.suspicious { + color: var(--arkham-warning); +} + +.lsb-item .value.success { + color: var(--arkham-success); +} + +.lsb-item .value.error { + color: var(--arkham-error); +} + +/* Magic Bytes */ +.hc-magic-bytes { + margin-bottom: 1.5rem; +} + +.magic-details { + display: flex; + flex-direction: column; + gap: 0.5rem; + padding: 1rem; + background: var(--arkham-bg-tertiary); + border-radius: var(--arkham-radius-sm); +} + +.magic-item { + display: flex; + justify-content: space-between; + align-items: center; +} + +.magic-item .label { + font-size: 0.875rem; + color: var(--arkham-text-secondary); +} + +.magic-item .value { + font-size: 0.875rem; + font-weight: 500; + color: var(--arkham-text-primary); + font-family: var(--arkham-font-mono); +} + +.magic-warning { + display: flex; + align-items: center; + gap: 0.5rem; + padding: 0.75rem; + background: rgba(248, 113, 113, 0.1); + border-radius: var(--arkham-radius-sm); + font-size: 0.875rem; + font-weight: 500; + color: var(--arkham-error); +} + +/* Recent Scans */ +.hc-recent-scans { + padding: 1.25rem; + background: var(--arkham-bg-secondary); + border: 1px solid var(--arkham-border); + border-radius: var(--arkham-radius-md); +} + +.hc-recent-scans h3 { + margin: 0 0 1rem 0; + font-size: 1rem; + font-weight: 600; + color: var(--arkham-text-primary); +} + +.recent-scans-list { + display: flex; + flex-direction: column; + gap: 0.5rem; +} + +.recent-scan-item { + display: flex; + align-items: center; + gap: 1rem; + padding: 0.625rem 0.875rem; + background: var(--arkham-bg-tertiary); + border: 1px solid var(--arkham-border); + border-radius: var(--arkham-radius-sm); + cursor: pointer; + transition: all 0.15s ease; +} + +.recent-scan-item:hover { + border-color: var(--arkham-accent-primary); +} + +.recent-scan-item.selected { + border-color: var(--arkham-accent-primary); + background: rgba(233, 69, 96, 0.05); +} + +.recent-scan-item .scan-doc { + font-size: 0.875rem; + font-family: var(--arkham-font-mono); + color: var(--arkham-text-primary); + flex: 1; +} + +.recent-scan-item .scan-confidence { + font-size: 0.8125rem; + font-weight: 600; + padding: 0.25rem 0.625rem; + border-radius: 12px; +} + +.scan-confidence.low { + background: rgba(74, 222, 128, 0.15); + color: var(--arkham-success); +} + +.scan-confidence.medium { + background: rgba(251, 191, 36, 0.15); + color: var(--arkham-warning); +} + +.scan-confidence.high { + background: rgba(251, 146, 60, 0.15); + color: #fb923c; +} + +.scan-confidence.critical { + background: rgba(248, 113, 113, 0.15); + color: var(--arkham-error); +} + +.recent-scan-item .scan-date { + font-size: 0.75rem; + color: var(--arkham-text-muted); +} + +/* Spin Animation */ +.spin { + animation: spin 1s linear infinite; +} + +@keyframes spin { + from { + transform: rotate(0deg); + } + to { + transform: rotate(360deg); + } +} + +/* Responsive for Hidden Content Tab */ +@media (max-width: 768px) { + .hc-stats-grid { + grid-template-columns: repeat(2, 1fr); + } + + .hc-scan-controls { + flex-direction: column; + } + + .hc-scan-form { + min-width: 100%; + } + + .hc-scan-actions { + width: 100%; + } + + .hc-scan-actions button { + flex: 1; + } + + .hc-result-summary { + flex-direction: column; + align-items: stretch; + } + + .hc-confidence-badge { + align-self: center; + } + + .indicators-list { + grid-template-columns: 1fr; + } + + .lsb-details { + grid-template-columns: repeat(2, 1fr); + } +} diff --git a/packages/arkham-shard-shell/src/pages/anomalies/AnomaliesPage.tsx b/packages/arkham-shard-shell/src/pages/anomalies/AnomaliesPage.tsx index 8616e99..c5e0b36 100644 --- a/packages/arkham-shard-shell/src/pages/anomalies/AnomaliesPage.tsx +++ b/packages/arkham-shard-shell/src/pages/anomalies/AnomaliesPage.tsx @@ -11,6 +11,7 @@ import { Icon } from '../../components/common/Icon'; import { LoadingSkeleton } from '../../components/common/LoadingSkeleton'; import { AIAnalystButton } from '../../components/AIAnalyst'; import { AnomalyDetail } from './AnomalyDetail'; +import { HiddenContentTab } from './HiddenContentTab'; import * as api from './api'; import './AnomaliesPage.css'; @@ -30,21 +31,61 @@ import { SEVERITY_OPTIONS, } from './types'; +// Tab types +type AnomaliesTab = 'list' | 'hidden-content'; + // ============================================ // Main Page Component // ============================================ export function AnomaliesPage() { - const [searchParams, _setSearchParams] = useSearchParams(); - void _setSearchParams; + const [searchParams, setSearchParams] = useSearchParams(); const anomalyId = searchParams.get('anomalyId'); + const tabParam = searchParams.get('tab') as AnomaliesTab | null; + const [activeTab, setActiveTab] = useState(tabParam || 'list'); // Show detail view if anomalyId is set if (anomalyId) { return ; } - return ; + const handleTabChange = (tab: AnomaliesTab) => { + setActiveTab(tab); + if (tab === 'list') { + setSearchParams({}); + } else { + setSearchParams({ tab }); + } + }; + + return ( +
+ {/* Tab Navigation */} +
+ + +
+ + {/* Tab Content */} + {activeTab === 'list' ? ( + + ) : ( + + )} +
+ ); } // ============================================ @@ -197,31 +238,25 @@ function AnomaliesListView() { }; if (loading && anomalies.length === 0) { - return ( -
- -
- ); + return ; } if (error) { return ( -
-
- -

Failed to load anomalies

-

{error}

- -
+
+ +

Failed to load anomalies

+

{error}

+
); } return ( -
+ <>
@@ -566,7 +601,7 @@ function AnomaliesListView() { onCancel={() => setShowDetectDialog(false)} /> )} -
+ ); } diff --git a/packages/arkham-shard-shell/src/pages/anomalies/HiddenContentTab.tsx b/packages/arkham-shard-shell/src/pages/anomalies/HiddenContentTab.tsx new file mode 100644 index 0000000..81c3f11 --- /dev/null +++ b/packages/arkham-shard-shell/src/pages/anomalies/HiddenContentTab.tsx @@ -0,0 +1,413 @@ +/** + * HiddenContentTab - Steganography and Hidden Data Detection + * + * Provides UI for detecting hidden content in documents: + * - Entropy analysis (encrypted/compressed data detection) + * - LSB pattern analysis (image steganography) + * - File type mismatch detection + * - Chi-square statistical tests + */ + +import { useState, useCallback } from 'react'; +import { Icon } from '../../components/common/Icon'; +import { useToast } from '../../context/ToastContext'; +import { useFetch } from '../../hooks/useFetch'; +import * as api from './api'; +import type { + HiddenContentScan, + HiddenContentStats, + EntropyRegion, + StegoIndicator, +} from './api'; + +// Props +interface HiddenContentTabProps { + onScanComplete?: () => void; +} + +export function HiddenContentTab({ onScanComplete }: HiddenContentTabProps) { + const { toast } = useToast(); + + // State + const [scanning, setScanning] = useState(false); + const [selectedDocId, setSelectedDocId] = useState(''); + const [selectedScan, setSelectedScan] = useState(null); + const [recentScans, setRecentScans] = useState([]); + + // Fetch stats + const { data: statsData, refetch: refetchStats } = useFetch<{ stats: HiddenContentStats }>( + '/api/anomalies/hidden-content/stats' + ); + const stats = statsData?.stats; + + // Fetch documents for selection + const { data: documentsData } = useFetch<{ items: Array<{ id: string; filename: string; file_type: string }> }>( + '/api/documents/items?limit=100' + ); + const documents = documentsData?.items || []; + + // Scan a document + const handleScan = useCallback(async () => { + if (!selectedDocId) { + toast.error('Please select a document to scan'); + return; + } + + setScanning(true); + try { + const result = await api.scanHiddenContent({ doc_id: selectedDocId }); + setSelectedScan(result.scan); + setRecentScans(prev => [result.scan, ...prev.slice(0, 9)]); + refetchStats(); + + if (result.anomaly_created) { + toast.warning('Hidden content detected! Anomaly created.'); + onScanComplete?.(); + } else if (result.scan.stego_confidence > 0.5) { + toast.warning(`Suspicious patterns detected (${(result.scan.stego_confidence * 100).toFixed(0)}% confidence)`); + } else { + toast.success('Scan complete - no hidden content detected'); + } + } catch (err) { + toast.error(err instanceof Error ? err.message : 'Scan failed'); + } finally { + setScanning(false); + } + }, [selectedDocId, toast, refetchStats, onScanComplete]); + + // Quick scan multiple documents + const handleQuickScan = useCallback(async () => { + const docIds = documents.slice(0, 20).map(d => d.id); + if (docIds.length === 0) { + toast.error('No documents available for scanning'); + return; + } + + setScanning(true); + try { + const result = await api.quickScanHiddenContent(docIds); + toast.success(`Scanned ${result.scanned} documents. ${result.requires_full_scan_count} need full analysis.`); + + if (result.requires_full_scan_count > 0) { + toast.warning(`High entropy detected in: ${result.requires_full_scan.join(', ')}`); + } + } catch (err) { + toast.error(err instanceof Error ? err.message : 'Quick scan failed'); + } finally { + setScanning(false); + } + }, [documents, toast]); + + return ( +
+ {/* Stats Overview */} +
+
+ +
{stats?.total_scans ?? 0}
+
Total Scans
+
+
+ +
{stats?.documents_with_findings ?? 0}
+
With Findings
+
+
+ +
{stats?.high_entropy_files ?? 0}
+
High Entropy
+
+
+ +
{stats?.stego_candidates ?? 0}
+
Stego Candidates
+
+
+ + {/* Scan Controls */} +
+
+ + +
+ +
+ + +
+
+ + {/* Scan Results */} + {selectedScan && ( +
+

+ + Scan Results +

+ + {/* Summary */} +
+
+ + {(selectedScan.stego_confidence * 100).toFixed(0)}% + + Stego Confidence +
+ +
+
+ Global Entropy: + 7 ? 'high' : selectedScan.entropy_global > 6 ? 'medium' : ''}`}> + {selectedScan.entropy_global.toFixed(3)} / 8.0 + +
+
+ File Type Match: + + {selectedScan.file_mismatch ? 'MISMATCH' : 'OK'} + +
+
+ Indicators Found: + {selectedScan.stego_indicators.length} +
+
+
+ + {/* Findings */} + {selectedScan.findings.length > 0 && ( +
+

+ + Findings +

+
    + {selectedScan.findings.map((finding, i) => ( +
  • {finding}
    • + ))} +
+
+ )} + + {/* Entropy Regions */} + {selectedScan.entropy_regions.length > 0 && ( +
+

+ + Entropy Analysis by Region +

+
+ {selectedScan.entropy_regions.map((region, i) => ( + + ))} +
+
+ )} + + {/* Stego Indicators */} + {selectedScan.stego_indicators.length > 0 && ( +
+

+ + Steganography Indicators +

+
+ {selectedScan.stego_indicators.map((indicator, i) => ( + + ))} +
+
+ )} + + {/* LSB Analysis */} + {selectedScan.lsb_result && ( +
+

+ + LSB (Least Significant Bit) Analysis +

+
+
+ Bit Ratio: + {(selectedScan.lsb_result.bit_ratio * 100).toFixed(2)}% +
+
+ Chi-Square Value: + {selectedScan.lsb_result.chi_square_value.toFixed(4)} +
+
+ P-Value: + + {selectedScan.lsb_result.chi_square_p_value.toFixed(6)} + +
+
+ Suspicious: + + {selectedScan.lsb_result.is_suspicious ? 'YES' : 'NO'} + +
+
+ Confidence: + {(selectedScan.lsb_result.confidence * 100).toFixed(0)}% +
+
+
+ )} + + {/* Magic Bytes */} +
+

+ + File Type Detection +

+
+
+ Expected Type: + {selectedScan.magic_expected || 'Unknown'} +
+
+ Actual Type: + {selectedScan.magic_actual || 'Unknown'} +
+ {selectedScan.file_mismatch && ( +
+ + File extension does not match actual file type! +
+ )} +
+
+
+ )} + + {/* Recent Scans */} + {recentScans.length > 0 && ( +
+

Recent Scans

+
+ {recentScans.map(scan => ( +
setSelectedScan(scan)} + > +
{scan.doc_id.slice(0, 8)}...
+
+ {(scan.stego_confidence * 100).toFixed(0)}% +
+
+ {new Date(scan.created_at).toLocaleTimeString()} +
+
+ ))} +
+
+ )} +
+ ); +} + +// Helper Components + +function EntropyRegionBar({ region, index }: { region: EntropyRegion; index: number }) { + const percentage = (region.entropy_value / 8) * 100; + const colorClass = region.is_anomalous ? 'anomalous' : percentage > 87.5 ? 'high' : percentage > 75 ? 'medium' : 'normal'; + + return ( +
+
+ Region {index + 1} + + ({region.start_offset} - {region.end_offset}) + +
+
+
+ {region.entropy_value.toFixed(3)} +
+ {region.description && ( +
+ {region.description} +
+ )} +
+ ); +} + +function StegoIndicatorCard({ indicator }: { indicator: StegoIndicator }) { + const iconMap: Record = { + entropy_spike: 'TrendingUp', + high_global_entropy: 'Thermometer', + lsb_anomaly: 'Binary', + histogram_anomaly: 'BarChart', + file_mismatch: 'FileWarning', + }; + + return ( +
0.7 ? 'high-conf' : ''}`}> +
+ +
+
+
+ {indicator.indicator_type.replace(/_/g, ' ')} +
+
+ Location: {indicator.location} +
+
+ {(indicator.confidence * 100).toFixed(0)}% confidence +
+
+
+ ); +} + +// Helper Functions + +function getConfidenceClass(confidence: number): string { + if (confidence >= 0.8) return 'critical'; + if (confidence >= 0.6) return 'high'; + if (confidence >= 0.4) return 'medium'; + return 'low'; +} + +export default HiddenContentTab; diff --git a/packages/arkham-shard-shell/src/pages/anomalies/api.ts b/packages/arkham-shard-shell/src/pages/anomalies/api.ts index 0420231..040d15d 100644 --- a/packages/arkham-shard-shell/src/pages/anomalies/api.ts +++ b/packages/arkham-shard-shell/src/pages/anomalies/api.ts @@ -172,3 +172,122 @@ export async function bulkUpdateStatus( }), }); } + +// ============================================ +// Hidden Content Detection Operations +// ============================================ + +export interface HiddenContentScanRequest { + doc_id: string; + scan_type?: string; + config?: Record; +} + +export interface EntropyRegion { + start_offset: number; + end_offset: number; + entropy_value: number; + is_anomalous: boolean; + description: string; +} + +export interface LSBAnalysisResult { + bit_ratio: number; + chi_square_value: number; + chi_square_p_value: number; + is_suspicious: boolean; + confidence: number; + sample_size: number; +} + +export interface StegoIndicator { + indicator_type: string; + confidence: number; + location: string; + details: Record; +} + +export interface HiddenContentScan { + id: string; + doc_id: string; + scan_type: string; + scan_status: string; + entropy_global: number; + entropy_regions: EntropyRegion[]; + magic_expected: string; + magic_actual: string; + file_mismatch: boolean; + lsb_result: LSBAnalysisResult | null; + stego_indicators: StegoIndicator[]; + stego_confidence: number; + findings: string[]; + created_at: string; + completed_at: string | null; +} + +export interface HiddenContentScanResponse { + scan: HiddenContentScan; + anomaly_created: boolean; +} + +export interface HiddenContentQuickScanRequest { + doc_ids: string[]; +} + +export interface QuickScanResult { + doc_id: string; + entropy_global?: number; + requires_full_scan?: boolean; + error?: string; +} + +export interface HiddenContentQuickScanResponse { + scanned: number; + results: QuickScanResult[]; + requires_full_scan_count: number; + requires_full_scan: string[]; +} + +export interface HiddenContentStats { + total_scans: number; + scans_by_type: Record; + documents_with_findings: number; + high_entropy_files: number; + stego_candidates: number; +} + +export async function scanHiddenContent( + request: HiddenContentScanRequest +): Promise { + return fetchAPI('/hidden-content/scan', { + method: 'POST', + body: JSON.stringify(request), + }); +} + +export async function quickScanHiddenContent( + docIds: string[] +): Promise { + return fetchAPI('/hidden-content/quick-scan', { + method: 'POST', + body: JSON.stringify({ doc_ids: docIds }), + }); +} + +export async function getHiddenContentScan( + scanId: string +): Promise<{ scan: HiddenContentScan }> { + return fetchAPI<{ scan: HiddenContentScan }>(`/hidden-content/${scanId}`); +} + +export async function getDocumentHiddenScans( + docId: string +): Promise<{ scans: HiddenContentScan[]; total: number }> { + return fetchAPI<{ scans: HiddenContentScan[]; total: number }>( + `/hidden-content/document/${docId}` + ); +} + +export async function getHiddenContentStats(): Promise<{ stats: HiddenContentStats }> { + return fetchAPI<{ stats: HiddenContentStats }>('/hidden-content/stats'); +} diff --git a/packages/arkham-shard-shell/src/pages/credibility/components/DeceptionPanel.tsx b/packages/arkham-shard-shell/src/pages/credibility/components/DeceptionPanel.tsx index 135f44b..3784c19 100644 --- a/packages/arkham-shard-shell/src/pages/credibility/components/DeceptionPanel.tsx +++ b/packages/arkham-shard-shell/src/pages/credibility/components/DeceptionPanel.tsx @@ -42,6 +42,7 @@ const CHECKLIST_TYPES: DeceptionChecklistType[] = ['mom', 'pop', 'moses', 'eve'] export function DeceptionPanel({ sourceType, sourceId, + credibilityAssessmentId, onRiskChange, }: DeceptionPanelProps) { const { toast } = useToast(); @@ -110,6 +111,8 @@ export function DeceptionPanel({ useEffect(() => { const fetchData = async () => { setLoading(true); + // Reset assessment state when source changes to avoid showing stale data + setAssessment(null); try { // Fetch source info for display await fetchSourceInfo(); @@ -161,9 +164,10 @@ export function DeceptionPanel({ source_type: sourceType, source_id: sourceId, source_name: sourceInfo?.name, + linked_assessment_id: credibilityAssessmentId, assessed_by: 'manual', affects_credibility: true, - credibility_weight: 0.3, + credibility_weight: 0.7, // Higher weight for meaningful impact }), }); diff --git a/packages/arkham-shard-shell/src/pages/documents/DeduplicationPanel.css b/packages/arkham-shard-shell/src/pages/documents/DeduplicationPanel.css new file mode 100644 index 0000000..e8ef4e3 --- /dev/null +++ b/packages/arkham-shard-shell/src/pages/documents/DeduplicationPanel.css @@ -0,0 +1,261 @@ +/* DeduplicationPanel Styles */ + +.dedup-panel { + display: flex; + flex-direction: column; + background: var(--surface-primary); + border: 1px solid var(--border-primary); + border-radius: 0.5rem; + overflow: hidden; +} + +/* Header */ +.dedup-header { + display: flex; + justify-content: space-between; + align-items: center; + padding: 1rem; + background: var(--surface-secondary); + border-bottom: 1px solid var(--border-primary); + gap: 1rem; +} + +.dedup-header-title { + display: flex; + align-items: center; + gap: 0.5rem; + min-width: 0; +} + +.dedup-header-title h3 { + margin: 0; + font-size: 0.9375rem; + font-weight: 600; + color: var(--text-primary); +} + +/* Badges */ +.dedup-badge { + padding: 0.125rem 0.5rem; + border-radius: 0.25rem; + font-size: 0.75rem; + font-weight: 500; +} + +.dedup-badge-warning { + background: rgba(245, 158, 11, 0.15); + color: #f59e0b; +} + +.dedup-badge-error { + background: rgba(239, 68, 68, 0.15); + color: #ef4444; +} + +/* Alert */ +.dedup-alert { + display: flex; + align-items: center; + gap: 0.5rem; + padding: 0.75rem 1rem; + font-size: 0.875rem; +} + +.dedup-alert-error { + background: rgba(239, 68, 68, 0.1); + border-bottom: 1px solid rgba(239, 68, 68, 0.2); + color: #ef4444; +} + +/* Sections */ +.dedup-section { + border-bottom: 1px solid var(--border-primary); +} + +.dedup-section:last-of-type { + border-bottom: none; +} + +.dedup-section-header { + width: 100%; + display: flex; + justify-content: space-between; + align-items: center; + padding: 0.75rem 1rem; + background: transparent; + border: none; + cursor: pointer; + transition: background 0.15s; +} + +.dedup-section-header:hover { + background: var(--surface-secondary); +} + +.dedup-section-title { + display: flex; + align-items: center; + gap: 0.5rem; + font-size: 0.875rem; + font-weight: 500; + color: var(--text-primary); +} + +/* Loading/Error/Empty states */ +.dedup-loading, +.dedup-error, +.dedup-empty { + display: flex; + align-items: center; + gap: 0.5rem; + padding: 1rem; + font-size: 0.875rem; + color: var(--text-secondary); +} + +.dedup-error { + color: #ef4444; +} + +.dedup-empty svg { + color: #10b981; +} + +/* Duplicate list */ +.dedup-list { + border-top: 1px solid var(--border-primary); +} + +.dedup-item { + display: flex; + align-items: center; + gap: 0.75rem; + padding: 0.75rem 1rem; + border-bottom: 1px solid var(--border-primary); + transition: background 0.15s; +} + +.dedup-item:last-child { + border-bottom: none; +} + +.dedup-item:hover { + background: var(--surface-secondary); +} + +.dedup-checkbox { + width: 1rem; + height: 1rem; + accent-color: var(--color-primary); + cursor: pointer; +} + +.dedup-item-info { + flex: 1; + min-width: 0; +} + +.dedup-item-title { + font-size: 0.875rem; + font-weight: 500; + color: var(--text-primary); + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.dedup-item-meta { + font-size: 0.75rem; + margin-top: 0.125rem; +} + +.dedup-exact { + color: #ef4444; +} + +.dedup-similar { + color: #f59e0b; +} + +/* Threshold control */ +.dedup-threshold { + padding: 0.75rem 1rem; + background: var(--surface-secondary); + border-top: 1px solid var(--border-primary); +} + +.dedup-threshold label { + display: block; + font-size: 0.75rem; + color: var(--text-tertiary); + margin-bottom: 0.375rem; +} + +.dedup-threshold-control { + display: flex; + align-items: center; + gap: 0.75rem; +} + +.dedup-threshold-control input[type="range"] { + flex: 1; + height: 4px; + accent-color: var(--color-primary); +} + +.dedup-threshold-control span { + font-size: 0.875rem; + font-weight: 500; + color: var(--text-primary); + min-width: 2.5rem; + text-align: right; +} + +/* Merge actions */ +.dedup-merge { + display: flex; + align-items: center; + gap: 0.75rem; + padding: 1rem; + background: var(--surface-secondary); + border-top: 1px solid var(--border-primary); + flex-wrap: wrap; +} + +.dedup-merge-count { + flex: 1; + font-size: 0.875rem; + color: var(--text-secondary); + min-width: 120px; +} + +.dedup-select { + padding: 0.375rem 0.75rem; + background: var(--surface-primary); + border: 1px solid var(--border-primary); + border-radius: 0.375rem; + color: var(--text-primary); + font-size: 0.875rem; + cursor: pointer; +} + +.dedup-select:focus { + outline: none; + border-color: var(--color-primary); +} + +/* Reuse btn styles from DocumentViewer.css - btn-primary addition */ +.btn-primary { + background: var(--color-primary); + color: white; + border: none; +} + +.btn-primary:hover { + background: var(--color-primary-hover, #2563eb); +} + +.btn-primary:disabled { + opacity: 0.5; + cursor: not-allowed; +} diff --git a/packages/arkham-shard-shell/src/pages/documents/DeduplicationPanel.tsx b/packages/arkham-shard-shell/src/pages/documents/DeduplicationPanel.tsx new file mode 100644 index 0000000..8b99c02 --- /dev/null +++ b/packages/arkham-shard-shell/src/pages/documents/DeduplicationPanel.tsx @@ -0,0 +1,291 @@ +/** + * Deduplication Panel + * + * Shows duplicate detection status and actions for a document. + */ + +import { useState, useCallback } from 'react'; +import { Icon } from '../../components/common/Icon'; +import { + useExactDuplicates, + useSimilarDuplicates, + useComputeHash, + useMergeDuplicates, +} from './api'; +import type { DuplicateResult, MergeRequest } from './api'; +import './DeduplicationPanel.css'; + +interface DeduplicationPanelProps { + documentId: string; + onViewDocument?: (docId: string) => void; + onRefresh?: () => void; +} + +export function DeduplicationPanel({ + documentId, + onViewDocument, + onRefresh, +}: DeduplicationPanelProps) { + const [showExact, setShowExact] = useState(true); + const [showSimilar, setShowSimilar] = useState(true); + const [similarThreshold, setSimilarThreshold] = useState(0.8); + const [selectedDuplicates, setSelectedDuplicates] = useState>(new Set()); + const [mergeStrategy, setMergeStrategy] = useState('soft_delete'); + + const { computeHash, loading: hashLoading, error: hashError } = useComputeHash(); + const { merge, loading: mergeLoading } = useMergeDuplicates(); + + const { + data: exactDuplicates, + loading: exactLoading, + error: exactError, + refetch: refetchExact, + } = useExactDuplicates(documentId); + + const { + data: similarDuplicates, + loading: similarLoading, + error: similarError, + refetch: refetchSimilar, + } = useSimilarDuplicates(documentId, similarThreshold); + + const handleComputeHash = useCallback(async () => { + try { + await computeHash(documentId); + refetchExact(); + refetchSimilar(); + onRefresh?.(); + } catch { + // Error handled by hook + } + }, [documentId, computeHash, refetchExact, refetchSimilar, onRefresh]); + + const toggleDuplicateSelection = useCallback((dupId: string) => { + setSelectedDuplicates((prev) => { + const next = new Set(prev); + if (next.has(dupId)) { + next.delete(dupId); + } else { + next.add(dupId); + } + return next; + }); + }, []); + + const handleMerge = useCallback(async () => { + if (selectedDuplicates.size === 0) return; + + try { + await merge({ + primary_id: documentId, + duplicate_ids: Array.from(selectedDuplicates), + cleanup_action: mergeStrategy, + preserve_references: true, + }); + setSelectedDuplicates(new Set()); + refetchExact(); + refetchSimilar(); + onRefresh?.(); + } catch { + // Error handled by hook + } + }, [selectedDuplicates, documentId, mergeStrategy, merge, refetchExact, refetchSimilar, onRefresh]); + + const renderDuplicateList = ( + duplicates: DuplicateResult[] | undefined, + loading: boolean, + error: Error | null, + type: 'exact' | 'similar' + ) => { + if (loading) { + return ( +
+ + Finding {type} duplicates... +
+ ); + } + + if (error) { + return ( +
+ + {error.message} +
+ ); + } + + if (!duplicates || duplicates.length === 0) { + return ( +
+ + No {type} duplicates found +
+ ); + } + + return ( +
+ {duplicates.map((dup) => ( +
+ toggleDuplicateSelection(dup.document_id)} + className="dedup-checkbox" + /> + +
+
{dup.title}
+
+ {type === 'exact' ? ( + Exact match (100%) + ) : ( + + {(dup.similarity_score * 100).toFixed(1)}% similar + (hamming: {dup.hamming_distance}) + + )} +
+
+ +
+ ))} +
+ ); + }; + + const totalDuplicates = + (exactDuplicates?.length || 0) + (similarDuplicates?.length || 0); + + return ( +
+ {/* Header */} +
+
+ +

Deduplication

+ {totalDuplicates > 0 && ( + + {totalDuplicates} found + + )} +
+ +
+ + {hashError && ( +
+ + {hashError.message} +
+ )} + + {/* Exact Duplicates Section */} +
+ + {showExact && renderDuplicateList(exactDuplicates ?? undefined, exactLoading, exactError, 'exact')} +
+ + {/* Similar Duplicates Section */} +
+ + {showSimilar && ( + <> +
+ +
+ setSimilarThreshold(parseFloat(e.target.value))} + /> + {(similarThreshold * 100).toFixed(0)}% +
+
+ {renderDuplicateList(similarDuplicates ?? undefined, similarLoading, similarError, 'similar')} + + )} +
+ + {/* Merge Actions */} + {selectedDuplicates.size > 0 && ( +
+ + {selectedDuplicates.size} selected for merge + + + +
+ )} +
+ ); +} + +export default DeduplicationPanel; diff --git a/packages/arkham-shard-shell/src/pages/documents/DocumentViewer.tsx b/packages/arkham-shard-shell/src/pages/documents/DocumentViewer.tsx index 4c7664e..17305b8 100644 --- a/packages/arkham-shard-shell/src/pages/documents/DocumentViewer.tsx +++ b/packages/arkham-shard-shell/src/pages/documents/DocumentViewer.tsx @@ -14,6 +14,7 @@ import { type DocumentChunk, type DocumentEntity, } from './api'; +import { DeduplicationPanel } from './DeduplicationPanel'; import './DocumentViewer.css'; interface DocumentViewerProps { @@ -21,7 +22,7 @@ interface DocumentViewerProps { onClose: () => void; } -type ViewerTab = 'content' | 'chunks' | 'entities' | 'metadata'; +type ViewerTab = 'content' | 'chunks' | 'entities' | 'metadata' | 'duplicates'; const ENTITY_TYPE_COLORS: Record = { PERSON: '#3b82f6', @@ -130,11 +131,29 @@ export function DocumentViewer({ documentId, onClose }: DocumentViewerProps) { return renderEntitiesTab(); case 'metadata': return renderMetadataTab(); + case 'duplicates': + return renderDuplicatesTab(); default: return null; } }; + const renderDuplicatesTab = () => { + return ( + { + // Could navigate to the duplicate document + console.log('View duplicate document:', docId); + }} + onRefresh={() => { + // Refresh document data after merge + refetchContent(); + }} + /> + ); + }; + const renderContentTab = () => { if (loadingContent) { return ( @@ -510,6 +529,13 @@ export function DocumentViewer({ documentId, onClose }: DocumentViewerProps) { Metadata +
{renderTabContent()}
diff --git a/packages/arkham-shard-shell/src/pages/documents/api.ts b/packages/arkham-shard-shell/src/pages/documents/api.ts index aa22678..5b72ed4 100644 --- a/packages/arkham-shard-shell/src/pages/documents/api.ts +++ b/packages/arkham-shard-shell/src/pages/documents/api.ts @@ -90,6 +90,39 @@ export interface BatchResult { details?: Array<{ id: string; success?: boolean; error?: string }>; } +// --- Deduplication Types --- + +export interface DocumentHash { + document_id: string; + content_md5: string; + content_sha256: string; + simhash: number; + text_length: number; +} + +export interface DuplicateResult { + document_id: string; + title: string; + similarity_score: number; + hamming_distance: number; + match_type: 'exact' | 'similar'; +} + +export interface DeduplicationStats { + total_documents: number; + documents_with_hash: number; + unique_content_hashes: number; + potential_duplicates: number; +} + +export interface MergeRequest { + primary_id: string; + duplicate_ids: string[]; + strategy?: string; + preserve_references?: boolean; + cleanup_action: 'soft_delete' | 'archive' | 'hard_delete' | 'keep'; +} + // --- API Functions --- async function fetchAPI(endpoint: string, options?: RequestInit): Promise { @@ -187,6 +220,51 @@ export async function batchDeleteDocuments(documentIds: string[]): Promise { + return fetchAPI(`/${documentId}/compute-hash`, { + method: 'POST', + }); +} + +export async function findExactDuplicates(documentId: string): Promise { + return fetchAPI(`/${documentId}/duplicates/exact`); +} + +export async function findSimilarDuplicates( + documentId: string, + threshold: number = 0.8 +): Promise { + return fetchAPI(`/${documentId}/duplicates/similar?threshold=${threshold}`); +} + +export async function getDeduplicationStats(): Promise { + return fetchAPI('/deduplication/stats'); +} + +export async function scanForDuplicates( + projectId: string, + similarityThreshold: number = 0.8 +): Promise { + return fetchAPI( + `/deduplication/scan?project_id=${projectId}`, + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ similarity_threshold: similarityThreshold }), + } + ); +} + +export async function mergeDuplicates(request: MergeRequest): Promise { + return fetchAPI('/deduplication/merge', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(request), + }); +} + // --- Hooks --- /** @@ -319,3 +397,80 @@ export function useBatchDelete() { return { deleteDocuments, loading, error }; } + +// --- Deduplication Hooks --- + +/** + * Hook for fetching deduplication stats + */ +export function useDeduplicationStats() { + return useFetch(`${API_PREFIX}/deduplication/stats`); +} + +/** + * Hook for finding exact duplicates of a document + */ +export function useExactDuplicates(documentId: string | null) { + const url = documentId ? `${API_PREFIX}/${documentId}/duplicates/exact` : null; + return useFetch(url); +} + +/** + * Hook for finding similar duplicates of a document + */ +export function useSimilarDuplicates(documentId: string | null, threshold: number = 0.8) { + const url = documentId + ? `${API_PREFIX}/${documentId}/duplicates/similar?threshold=${threshold}` + : null; + return useFetch(url); +} + +/** + * Hook for computing document hash + */ +export function useComputeHash() { + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + + const computeHash = useCallback(async (documentId: string) => { + setLoading(true); + setError(null); + try { + const result = await computeDocumentHash(documentId); + return result; + } catch (err) { + const error = err instanceof Error ? err : new Error('Hash computation failed'); + setError(error); + throw error; + } finally { + setLoading(false); + } + }, []); + + return { computeHash, loading, error }; +} + +/** + * Hook for merging duplicates + */ +export function useMergeDuplicates() { + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + + const merge = useCallback(async (request: MergeRequest) => { + setLoading(true); + setError(null); + try { + const result = await mergeDuplicates(request); + return result; + } catch (err) { + const error = err instanceof Error ? err : new Error('Merge failed'); + setError(error); + throw error; + } finally { + setLoading(false); + } + }, []); + + return { merge, loading, error }; +} diff --git a/packages/arkham-shard-shell/src/pages/embed/EmbedPage.tsx b/packages/arkham-shard-shell/src/pages/embed/EmbedPage.tsx index 09bed89..4c3ee5c 100644 --- a/packages/arkham-shard-shell/src/pages/embed/EmbedPage.tsx +++ b/packages/arkham-shard-shell/src/pages/embed/EmbedPage.tsx @@ -691,12 +691,12 @@ export function EmbedPage() { )} - {/* Cache Statistics */} + {/* Embedding Statistics */}

- Cache Statistics + Embedding Statistics

) : cacheStats ? (
- +
{formatNumber(cacheStats.hits)}
-
Hits
+
Embeddings
- +
{formatNumber(cacheStats.misses)}
-
Misses
+
Pending
@@ -744,24 +744,24 @@ export function EmbedPage() {
{(cacheStats.hit_rate * 100).toFixed(1)}%
-
Hit Rate
+
Coverage
- +
{cacheStats.size}/{cacheStats.max_size}
-
Cache Size
+
Docs / Chunks
) : (
-

No cache statistics available

+

No statistics available

)} diff --git a/packages/arkham-shard-shell/src/pages/provenance/ForensicsTab.tsx b/packages/arkham-shard-shell/src/pages/provenance/ForensicsTab.tsx new file mode 100644 index 0000000..870e722 --- /dev/null +++ b/packages/arkham-shard-shell/src/pages/provenance/ForensicsTab.tsx @@ -0,0 +1,776 @@ +/** + * ForensicsTab - Metadata Forensics Analysis + * + * Provides UI for forensic metadata analysis: + * - EXIF extraction (camera info, GPS, timestamps) + * - PDF metadata extraction + * - Office document metadata + * - Timeline reconstruction + * - Document comparison + */ + +import { useState, useCallback } from 'react'; +import { Icon } from '../../components/common/Icon'; +import { useToast } from '../../context/ToastContext'; +import { useFetch } from '../../hooks/useFetch'; + +// Types +interface ExifData { + make?: string; + model?: string; + serial_number?: string; + datetime_original?: string; + datetime_digitized?: string; + datetime_modified?: string; + gps_latitude?: number; + gps_longitude?: number; + gps_altitude?: number; + software?: string; + width?: number; + height?: number; +} + +interface PdfMetadata { + title?: string; + author?: string; + subject?: string; + creator?: string; + producer?: string; + creation_date?: string; + modification_date?: string; + keywords?: string[]; + page_count?: number; + pdf_version?: string; + is_encrypted?: boolean; +} + +interface OfficeMetadata { + title?: string; + author?: string; + subject?: string; + company?: string; + manager?: string; + created?: string; + modified?: string; + last_modified_by?: string; + revision?: number; + keywords?: string[]; + category?: string; +} + +interface ForensicFinding { + finding_type: string; + severity: string; + description: string; + evidence: Record; + confidence: number; +} + +interface TimelineEvent { + id: string; + doc_id: string; + event_type: string; + event_timestamp?: string; + event_source: string; + event_actor?: string; + event_details: Record; + confidence: number; + is_estimated: boolean; +} + +interface ForensicScan { + id: string; + doc_id: string; + scan_status: string; + file_hash_md5?: string; + file_hash_sha256?: string; + file_hash_sha512?: string; + file_size?: number; + exif_data?: ExifData; + pdf_metadata?: PdfMetadata; + office_metadata?: OfficeMetadata; + findings: ForensicFinding[]; + integrity_status: string; + confidence_score: number; + timeline_events: TimelineEvent[]; + scanned_at?: string; +} + +interface ForensicStats { + total_scans: number; + scans_by_status: Record; + documents_with_findings: number; + integrity_status_counts: Record; + average_confidence: number; +} + +interface ForensicsTabProps { + onScanComplete?: () => void; +} + +export function ForensicsTab({ onScanComplete }: ForensicsTabProps) { + const { toast } = useToast(); + + // State + const [scanning, setScanning] = useState(false); + const [selectedDocId, setSelectedDocId] = useState(''); + const [selectedScan, setSelectedScan] = useState(null); + const [recentScans, setRecentScans] = useState([]); + const [comparing, setComparing] = useState(false); + const [compareSourceId, setCompareSourceId] = useState(''); + const [compareTargetId, setCompareTargetId] = useState(''); + const [comparisonResult, setComparisonResult] = useState(null); + + // Fetch stats + const { data: statsData, refetch: refetchStats } = useFetch<{ stats: ForensicStats }>( + '/api/provenance/forensics/stats' + ); + const stats = statsData?.stats; + + // Fetch documents for selection + const { data: documentsData } = useFetch<{ items: Array<{ id: string; filename: string; file_type: string }> }>( + '/api/documents/items?limit=100' + ); + const documents = documentsData?.items || []; + + // Scan a document + const handleScan = useCallback(async () => { + if (!selectedDocId) { + toast.error('Please select a document to scan'); + return; + } + + setScanning(true); + try { + const response = await fetch('/api/provenance/forensics/scan', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ doc_id: selectedDocId }), + }); + + if (!response.ok) { + const error = await response.json(); + throw new Error(error.detail || 'Scan failed'); + } + + const result = await response.json(); + setSelectedScan(result.scan); + setRecentScans(prev => [result.scan, ...prev.slice(0, 9)]); + refetchStats(); + + if (result.scan.findings.length > 0) { + toast.warning(`Forensic analysis complete. ${result.scan.findings.length} findings detected.`); + } else { + toast.success('Forensic analysis complete - no issues detected'); + } + onScanComplete?.(); + } catch (err) { + toast.error(err instanceof Error ? err.message : 'Scan failed'); + } finally { + setScanning(false); + } + }, [selectedDocId, toast, refetchStats, onScanComplete]); + + // Compare documents + const handleCompare = useCallback(async () => { + if (!compareSourceId || !compareTargetId) { + toast.error('Please select both source and target documents'); + return; + } + + setComparing(true); + try { + const response = await fetch('/api/provenance/forensics/compare', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + source_doc_id: compareSourceId, + target_doc_id: compareTargetId, + }), + }); + + if (!response.ok) { + const error = await response.json(); + throw new Error(error.detail || 'Comparison failed'); + } + + const result = await response.json(); + setComparisonResult(result.comparison); + toast.success('Document comparison complete'); + } catch (err) { + toast.error(err instanceof Error ? err.message : 'Comparison failed'); + } finally { + setComparing(false); + } + }, [compareSourceId, compareTargetId, toast]); + + const formatDate = (dateStr?: string) => { + if (!dateStr) return 'N/A'; + try { + return new Date(dateStr).toLocaleString(); + } catch { + return dateStr; + } + }; + + const getIntegrityBadgeClass = (status: string): string => { + switch (status) { + case 'pristine': return 'integrity-pristine'; + case 'modified': return 'integrity-modified'; + case 'suspicious': return 'integrity-suspicious'; + case 'compromised': return 'integrity-compromised'; + default: return 'integrity-unknown'; + } + }; + + const getSeverityClass = (severity: string): string => { + switch (severity) { + case 'critical': return 'severity-critical'; + case 'high': return 'severity-high'; + case 'medium': return 'severity-medium'; + case 'low': return 'severity-low'; + default: return 'severity-info'; + } + }; + + return ( +
+ {/* Stats Overview */} +
+
+ +
{stats?.total_scans ?? 0}
+
Total Scans
+
+
+ +
{stats?.documents_with_findings ?? 0}
+
With Findings
+
+
+ +
{stats?.integrity_status_counts?.pristine ?? 0}
+
Pristine
+
+
+ +
+ {stats?.average_confidence ? `${(stats.average_confidence * 100).toFixed(0)}%` : '0%'} +
+
Avg Confidence
+
+
+ + {/* Scan Controls */} +
+
+ + +
+ +
+ +
+
+ + {/* Scan Results */} + {selectedScan && ( +
+

+ + Forensic Analysis Results +

+ + {/* Summary */} +
+
+ {selectedScan.integrity_status} + Integrity Status +
+ +
+
+ Confidence: + {(selectedScan.confidence_score * 100).toFixed(0)}% +
+
+ Findings: + 0 ? 'warning' : ''}`}> + {selectedScan.findings.length} + +
+
+ Timeline Events: + {selectedScan.timeline_events.length} +
+
+ File Size: + + {selectedScan.file_size ? `${(selectedScan.file_size / 1024).toFixed(1)} KB` : 'N/A'} + +
+
+
+ + {/* File Hashes */} + {(selectedScan.file_hash_md5 || selectedScan.file_hash_sha256) && ( +
+

+ + File Hashes +

+
+ {selectedScan.file_hash_md5 && ( +
+ MD5: + {selectedScan.file_hash_md5} +
+ )} + {selectedScan.file_hash_sha256 && ( +
+ SHA256: + {selectedScan.file_hash_sha256.substring(0, 32)}... +
+ )} +
+
+ )} + + {/* EXIF Data */} + {selectedScan.exif_data && Object.values(selectedScan.exif_data).some(v => v !== null && v !== undefined) && ( +
+

+ + EXIF Data (Image Metadata) +

+
+ {selectedScan.exif_data.make && ( +
+ Camera Make: + {selectedScan.exif_data.make} +
+ )} + {selectedScan.exif_data.model && ( +
+ Camera Model: + {selectedScan.exif_data.model} +
+ )} + {selectedScan.exif_data.serial_number && ( +
+ Serial Number: + {selectedScan.exif_data.serial_number} +
+ )} + {selectedScan.exif_data.software && ( +
+ Software: + {selectedScan.exif_data.software} +
+ )} + {selectedScan.exif_data.datetime_original && ( +
+ Date Taken: + {formatDate(selectedScan.exif_data.datetime_original)} +
+ )} + {(selectedScan.exif_data.gps_latitude || selectedScan.exif_data.gps_longitude) && ( +
+ GPS Location: + + {selectedScan.exif_data.gps_latitude?.toFixed(6)}, {selectedScan.exif_data.gps_longitude?.toFixed(6)} + +
+ )} + {(selectedScan.exif_data.width && selectedScan.exif_data.height) && ( +
+ Dimensions: + {selectedScan.exif_data.width} x {selectedScan.exif_data.height} +
+ )} +
+
+ )} + + {/* PDF Metadata */} + {selectedScan.pdf_metadata && Object.values(selectedScan.pdf_metadata).some(v => v !== null && v !== undefined) && ( +
+

+ + PDF Metadata +

+
+ {selectedScan.pdf_metadata.title && ( +
+ Title: + {selectedScan.pdf_metadata.title} +
+ )} + {selectedScan.pdf_metadata.author && ( +
+ Author: + {selectedScan.pdf_metadata.author} +
+ )} + {selectedScan.pdf_metadata.creator && ( +
+ Creator: + {selectedScan.pdf_metadata.creator} +
+ )} + {selectedScan.pdf_metadata.producer && ( +
+ Producer: + {selectedScan.pdf_metadata.producer} +
+ )} + {selectedScan.pdf_metadata.creation_date && ( +
+ Created: + {formatDate(selectedScan.pdf_metadata.creation_date)} +
+ )} + {selectedScan.pdf_metadata.modification_date && ( +
+ Modified: + {formatDate(selectedScan.pdf_metadata.modification_date)} +
+ )} + {selectedScan.pdf_metadata.page_count && ( +
+ Pages: + {selectedScan.pdf_metadata.page_count} +
+ )} + {selectedScan.pdf_metadata.pdf_version && ( +
+ PDF Version: + {selectedScan.pdf_metadata.pdf_version} +
+ )} + {selectedScan.pdf_metadata.is_encrypted !== undefined && ( +
+ Encrypted: + + {selectedScan.pdf_metadata.is_encrypted ? 'Yes' : 'No'} + +
+ )} + {selectedScan.pdf_metadata.keywords && selectedScan.pdf_metadata.keywords.length > 0 && ( +
+ Keywords: + {selectedScan.pdf_metadata.keywords.join(', ')} +
+ )} +
+
+ )} + + {/* Office Metadata */} + {selectedScan.office_metadata && Object.values(selectedScan.office_metadata).some(v => v !== null && v !== undefined) && ( +
+

+ + Office Document Metadata +

+
+ {selectedScan.office_metadata.title && ( +
+ Title: + {selectedScan.office_metadata.title} +
+ )} + {selectedScan.office_metadata.author && ( +
+ Author: + {selectedScan.office_metadata.author} +
+ )} + {selectedScan.office_metadata.company && ( +
+ Company: + {selectedScan.office_metadata.company} +
+ )} + {selectedScan.office_metadata.last_modified_by && ( +
+ Last Modified By: + {selectedScan.office_metadata.last_modified_by} +
+ )} + {selectedScan.office_metadata.created && ( +
+ Created: + {formatDate(selectedScan.office_metadata.created)} +
+ )} + {selectedScan.office_metadata.modified && ( +
+ Modified: + {formatDate(selectedScan.office_metadata.modified)} +
+ )} + {selectedScan.office_metadata.revision && ( +
+ Revision: + {selectedScan.office_metadata.revision} +
+ )} + {selectedScan.office_metadata.subject && ( +
+ Subject: + {selectedScan.office_metadata.subject} +
+ )} + {selectedScan.office_metadata.keywords && selectedScan.office_metadata.keywords.length > 0 && ( +
+ Keywords: + {selectedScan.office_metadata.keywords.join(', ')} +
+ )} +
+
+ )} + + {/* Findings */} + {selectedScan.findings.length > 0 && ( +
+

+ + Findings ({selectedScan.findings.length}) +

+
+ {selectedScan.findings.map((finding, i) => ( +
+
+ {finding.finding_type.replace(/_/g, ' ')} + + {finding.severity} + +
+

{finding.description}

+
+ {(finding.confidence * 100).toFixed(0)}% confidence +
+
+ ))} +
+
+ )} + + {/* Timeline Events */} + {selectedScan.timeline_events.length > 0 && ( +
+

+ + Timeline Reconstruction ({selectedScan.timeline_events.length} events) +

+
+ {selectedScan.timeline_events + .sort((a, b) => { + if (!a.event_timestamp) return 1; + if (!b.event_timestamp) return -1; + return new Date(a.event_timestamp).getTime() - new Date(b.event_timestamp).getTime(); + }) + .map((event, i) => ( +
+
+ +
+
+
+ {event.event_type.replace(/_/g, ' ')} + {event.is_estimated && Estimated} +
+
+ {event.event_timestamp ? formatDate(event.event_timestamp) : 'Unknown time'} +
+ {event.event_actor && ( +
+ + {event.event_actor} +
+ )} +
+ Source: {event.event_source} +
+
+ {(event.confidence * 100).toFixed(0)}% confidence +
+
+
+ ))} +
+
+ )} +
+ )} + + {/* Document Comparison */} +
+

+ + Document Comparison +

+
+
+ + +
+
+ +
+
+ + +
+ +
+ + {/* Comparison Result */} + {comparisonResult && ( +
+
+
+ {(comparisonResult.match_score * 100).toFixed(0)}% + Match Score +
+
+ Relationship: + + {comparisonResult.relationship_type?.replace(/_/g, ' ') || 'Unknown'} + +
+
+ Confidence: + {(comparisonResult.confidence * 100).toFixed(0)}% +
+
+ + {comparisonResult.similarities && comparisonResult.similarities.length > 0 && ( +
+

+ + Similarities ({comparisonResult.similarities.length}) +

+
    + {comparisonResult.similarities.map((item: string, i: number) => ( +
  • {item}
    • + ))} +
+
+ )} + + {comparisonResult.differences && comparisonResult.differences.length > 0 && ( +
+

+ + Differences ({comparisonResult.differences.length}) +

+
    + {comparisonResult.differences.map((item: string, i: number) => ( +
  • {item}
    • + ))} +
+
+ )} +
+ )} +
+ + {/* Recent Scans */} + {recentScans.length > 0 && ( +
+

Recent Scans

+
+ {recentScans.map(scan => ( +
setSelectedScan(scan)} + > +
{scan.doc_id.slice(0, 8)}...
+
+ {scan.integrity_status} +
+
+ {scan.findings.length} findings +
+
+ {scan.scanned_at ? new Date(scan.scanned_at).toLocaleTimeString() : 'N/A'} +
+
+ ))} +
+
+ )} +
+ ); +} + +export default ForensicsTab; diff --git a/packages/arkham-shard-shell/src/pages/provenance/ProvenancePage.css b/packages/arkham-shard-shell/src/pages/provenance/ProvenancePage.css index c71a6a5..7726d04 100644 --- a/packages/arkham-shard-shell/src/pages/provenance/ProvenancePage.css +++ b/packages/arkham-shard-shell/src/pages/provenance/ProvenancePage.css @@ -757,3 +757,809 @@ transform: rotate(360deg); } } + +/* ============================================ + Forensics Tab + ============================================ */ + +.forensics-tab { + display: flex; + flex-direction: column; + gap: 24px; + padding: 24px; + overflow-y: auto; + height: 100%; +} + +/* Stats Grid */ +.forensics-stats-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(160px, 1fr)); + gap: 16px; +} + +.forensics-stat-card { + display: flex; + flex-direction: column; + align-items: center; + gap: 8px; + padding: 20px; + background: var(--color-surface); + border: 1px solid var(--color-border); + border-radius: 12px; + text-align: center; +} + +.forensics-stat-card svg { + color: var(--color-primary); + opacity: 0.8; +} + +.forensics-stat-value { + font-size: 28px; + font-weight: 700; + color: var(--color-text); +} + +.forensics-stat-label { + font-size: 12px; + color: var(--color-text-secondary); + text-transform: uppercase; + letter-spacing: 0.5px; +} + +/* Scan Controls */ +.forensics-scan-controls { + display: flex; + gap: 16px; + padding: 20px; + background: var(--color-surface); + border: 1px solid var(--color-border); + border-radius: 12px; + flex-wrap: wrap; + align-items: flex-end; +} + +.forensics-scan-form { + display: flex; + flex-direction: column; + gap: 8px; + flex: 1; + min-width: 250px; +} + +.forensics-scan-form label { + font-size: 14px; + font-weight: 500; + color: var(--color-text-secondary); +} + +.forensics-scan-form select { + padding: 10px 12px; + border: 1px solid var(--color-border); + border-radius: 6px; + background: var(--color-background); + color: var(--color-text); + font-size: 14px; +} + +.forensics-scan-form select:focus { + outline: none; + border-color: var(--color-primary); +} + +.forensics-scan-actions { + display: flex; + gap: 8px; +} + +/* Scan Results */ +.forensics-scan-results { + padding: 24px; + background: var(--color-surface); + border: 1px solid var(--color-border); + border-radius: 12px; +} + +.forensics-scan-results h3 { + display: flex; + align-items: center; + gap: 8px; + margin: 0 0 20px 0; + font-size: 18px; + font-weight: 600; + color: var(--color-text); +} + +.forensics-scan-results h4 { + display: flex; + align-items: center; + gap: 8px; + margin: 0 0 12px 0; + font-size: 15px; + font-weight: 600; + color: var(--color-text); +} + +/* Result Summary */ +.forensics-result-summary { + display: flex; + gap: 24px; + padding: 16px; + background: var(--color-background); + border-radius: 8px; + margin-bottom: 24px; + flex-wrap: wrap; + align-items: center; +} + +.forensics-integrity-badge { + display: flex; + flex-direction: column; + align-items: center; + padding: 16px 24px; + border-radius: 8px; + min-width: 120px; +} + +.forensics-integrity-badge .integrity-status { + font-size: 20px; + font-weight: 700; + text-transform: capitalize; +} + +.forensics-integrity-badge .integrity-label { + font-size: 11px; + text-transform: uppercase; + letter-spacing: 0.5px; + opacity: 0.9; +} + +.integrity-pristine { + background: rgba(74, 222, 128, 0.15); + color: var(--color-success); +} + +.integrity-modified { + background: rgba(251, 191, 36, 0.15); + color: var(--color-warning, #fbbf24); +} + +.integrity-suspicious { + background: rgba(251, 146, 60, 0.15); + color: #fb923c; +} + +.integrity-compromised { + background: rgba(248, 113, 113, 0.15); + color: var(--color-danger); +} + +.integrity-unknown { + background: var(--color-surface); + color: var(--color-text-secondary); +} + +.forensics-summary-stats { + display: flex; + flex-direction: column; + gap: 8px; + flex: 1; +} + +.forensics-summary-stats .summary-item { + display: flex; + justify-content: space-between; + padding: 6px 0; + border-bottom: 1px solid var(--color-border); +} + +.forensics-summary-stats .summary-item:last-child { + border-bottom: none; +} + +.forensics-summary-stats .label { + font-size: 14px; + color: var(--color-text-secondary); +} + +.forensics-summary-stats .value { + font-size: 14px; + font-weight: 600; + color: var(--color-text); +} + +.forensics-summary-stats .value.warning { + color: var(--color-warning, #fbbf24); +} + +/* File Hashes */ +.forensics-hashes { + margin-bottom: 24px; +} + +.hash-grid { + display: flex; + flex-direction: column; + gap: 8px; + padding: 12px; + background: var(--color-background); + border-radius: 6px; +} + +.hash-item { + display: flex; + gap: 12px; + align-items: center; +} + +.hash-label { + font-size: 13px; + font-weight: 500; + color: var(--color-text-secondary); + min-width: 60px; +} + +.hash-value { + font-family: 'Courier New', monospace; + font-size: 12px; + padding: 4px 8px; + background: var(--color-surface); + border-radius: 4px; + color: var(--color-text); + word-break: break-all; +} + +/* Metadata Sections */ +.forensics-exif, +.forensics-pdf, +.forensics-office { + margin-bottom: 24px; +} + +.metadata-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(200px, 1fr)); + gap: 12px; + padding: 16px; + background: var(--color-background); + border-radius: 8px; +} + +.metadata-item { + display: flex; + flex-direction: column; + gap: 4px; +} + +.metadata-item.full-width { + grid-column: 1 / -1; +} + +.metadata-item .label { + font-size: 12px; + color: var(--color-text-secondary); + text-transform: uppercase; + letter-spacing: 0.5px; +} + +.metadata-item .value { + font-size: 14px; + color: var(--color-text); + word-break: break-word; +} + +.metadata-item .value.warning { + color: var(--color-warning, #fbbf24); +} + +/* Findings */ +.forensics-findings { + margin-bottom: 24px; +} + +.findings-list { + display: flex; + flex-direction: column; + gap: 12px; +} + +.finding-card { + padding: 16px; + background: var(--color-background); + border: 1px solid var(--color-border); + border-radius: 8px; + border-left: 4px solid; +} + +.finding-card.severity-critical { + border-left-color: var(--color-danger); +} + +.finding-card.severity-high { + border-left-color: #fb923c; +} + +.finding-card.severity-medium { + border-left-color: var(--color-warning, #fbbf24); +} + +.finding-card.severity-low { + border-left-color: var(--color-text-secondary); +} + +.finding-card.severity-info { + border-left-color: var(--color-primary); +} + +.finding-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 8px; +} + +.finding-type { + font-weight: 600; + color: var(--color-text); + text-transform: capitalize; +} + +.finding-severity { + padding: 4px 8px; + border-radius: 4px; + font-size: 12px; + font-weight: 500; + text-transform: uppercase; +} + +.finding-severity.severity-critical { + background: rgba(248, 113, 113, 0.15); + color: var(--color-danger); +} + +.finding-severity.severity-high { + background: rgba(251, 146, 60, 0.15); + color: #fb923c; +} + +.finding-severity.severity-medium { + background: rgba(251, 191, 36, 0.15); + color: var(--color-warning, #fbbf24); +} + +.finding-severity.severity-low { + background: var(--color-surface); + color: var(--color-text-secondary); +} + +.finding-description { + margin: 0 0 8px 0; + font-size: 14px; + color: var(--color-text); + line-height: 1.5; +} + +.finding-confidence { + font-size: 12px; + color: var(--color-text-secondary); +} + +/* Timeline */ +.forensics-timeline { + margin-bottom: 24px; +} + +.timeline-list { + display: flex; + flex-direction: column; + gap: 0; + position: relative; + padding-left: 24px; +} + +.timeline-list::before { + content: ''; + position: absolute; + left: 4px; + top: 8px; + bottom: 8px; + width: 2px; + background: var(--color-border); +} + +.timeline-event { + display: flex; + gap: 16px; + padding: 12px 0; + position: relative; +} + +.timeline-event.estimated { + opacity: 0.7; +} + +.event-marker { + position: absolute; + left: -24px; + top: 14px; + width: 10px; + height: 10px; + background: var(--color-primary); + border-radius: 50%; + border: 2px solid var(--color-background); +} + +.event-content { + display: flex; + flex-direction: column; + gap: 4px; + flex: 1; +} + +.event-header { + display: flex; + align-items: center; + gap: 8px; +} + +.event-type { + font-weight: 600; + font-size: 14px; + color: var(--color-text); + text-transform: capitalize; +} + +.event-estimated { + font-size: 11px; + padding: 2px 6px; + background: var(--color-surface); + border-radius: 4px; + color: var(--color-text-secondary); +} + +.event-timestamp { + font-size: 13px; + color: var(--color-text); +} + +.event-actor { + display: flex; + align-items: center; + gap: 4px; + font-size: 12px; + color: var(--color-text-secondary); +} + +.event-source { + font-size: 12px; + color: var(--color-text-secondary); +} + +.event-confidence { + font-size: 11px; + color: var(--color-text-secondary); +} + +/* Document Comparison */ +.forensics-comparison { + padding: 24px; + background: var(--color-surface); + border: 1px solid var(--color-border); + border-radius: 12px; +} + +.forensics-comparison h3 { + display: flex; + align-items: center; + gap: 8px; + margin: 0 0 20px 0; + font-size: 18px; + font-weight: 600; + color: var(--color-text); +} + +.comparison-controls { + display: flex; + gap: 12px; + align-items: flex-end; + flex-wrap: wrap; +} + +.comparison-select { + display: flex; + flex-direction: column; + gap: 6px; + flex: 1; + min-width: 180px; +} + +.comparison-select label { + font-size: 13px; + color: var(--color-text-secondary); +} + +.comparison-select select { + padding: 10px 12px; + border: 1px solid var(--color-border); + border-radius: 6px; + background: var(--color-background); + color: var(--color-text); + font-size: 14px; +} + +.comparison-arrow { + display: flex; + align-items: center; + justify-content: center; + padding: 10px; + color: var(--color-text-secondary); +} + +/* Comparison Result */ +.comparison-result { + margin-top: 24px; + padding: 16px; + background: var(--color-background); + border-radius: 8px; +} + +.comparison-header { + display: flex; + gap: 24px; + align-items: center; + flex-wrap: wrap; + padding-bottom: 16px; + border-bottom: 1px solid var(--color-border); + margin-bottom: 16px; +} + +.match-score { + display: flex; + flex-direction: column; + align-items: center; +} + +.match-score .score-value { + font-size: 32px; + font-weight: 700; + color: var(--color-primary); +} + +.match-score .score-label { + font-size: 11px; + text-transform: uppercase; + color: var(--color-text-secondary); +} + +.relationship-type { + display: flex; + flex-direction: column; + gap: 4px; +} + +.relationship-label { + font-size: 11px; + text-transform: uppercase; + color: var(--color-text-secondary); +} + +.relationship-value { + font-size: 16px; + font-weight: 600; + text-transform: capitalize; + color: var(--color-text); +} + +.relationship-value.copy { + color: var(--color-danger); +} + +.relationship-value.same_source { + color: #fb923c; +} + +.relationship-value.same_author { + color: var(--color-warning, #fbbf24); +} + +.comparison-confidence { + display: flex; + flex-direction: column; + gap: 4px; +} + +.confidence-label { + font-size: 11px; + text-transform: uppercase; + color: var(--color-text-secondary); +} + +.confidence-value { + font-size: 16px; + font-weight: 600; + color: var(--color-text); +} + +.comparison-section { + margin-bottom: 16px; +} + +.comparison-section:last-child { + margin-bottom: 0; +} + +.comparison-section h4 { + display: flex; + align-items: center; + gap: 8px; + font-size: 14px; + font-weight: 600; + margin: 0 0 8px 0; + color: var(--color-text); +} + +.comparison-list { + margin: 0; + padding-left: 24px; +} + +.comparison-list li { + font-size: 14px; + color: var(--color-text); + margin-bottom: 4px; + line-height: 1.5; +} + +.comparison-list.similarities li::marker { + color: var(--color-success); +} + +.comparison-list.differences li::marker { + color: var(--color-danger); +} + +/* Recent Scans */ +.forensics-recent-scans { + padding: 20px; + background: var(--color-surface); + border: 1px solid var(--color-border); + border-radius: 12px; +} + +.forensics-recent-scans h3 { + margin: 0 0 16px 0; + font-size: 16px; + font-weight: 600; + color: var(--color-text); +} + +.recent-scans-list { + display: flex; + flex-direction: column; + gap: 8px; +} + +.recent-scan-item { + display: flex; + align-items: center; + gap: 16px; + padding: 10px 14px; + background: var(--color-background); + border: 1px solid var(--color-border); + border-radius: 6px; + cursor: pointer; + transition: all 0.2s; +} + +.recent-scan-item:hover { + border-color: var(--color-primary); +} + +.recent-scan-item.selected { + border-color: var(--color-primary); + background: rgba(var(--color-primary-rgb), 0.05); +} + +.recent-scan-item .scan-doc { + font-size: 14px; + font-family: 'Courier New', monospace; + color: var(--color-text); + flex: 1; +} + +.recent-scan-item .scan-integrity { + font-size: 12px; + font-weight: 500; + padding: 4px 8px; + border-radius: 4px; + text-transform: capitalize; +} + +.scan-integrity.integrity-pristine { + background: rgba(74, 222, 128, 0.15); + color: var(--color-success); +} + +.scan-integrity.integrity-modified { + background: rgba(251, 191, 36, 0.15); + color: var(--color-warning, #fbbf24); +} + +.scan-integrity.integrity-suspicious { + background: rgba(251, 146, 60, 0.15); + color: #fb923c; +} + +.scan-integrity.integrity-compromised { + background: rgba(248, 113, 113, 0.15); + color: var(--color-danger); +} + +.recent-scan-item .scan-findings { + font-size: 12px; + color: var(--color-text-secondary); +} + +.recent-scan-item .scan-date { + font-size: 12px; + color: var(--color-text-secondary); +} + +/* Responsive */ +@media (max-width: 768px) { + .forensics-tab { + padding: 16px; + } + + .forensics-stats-grid { + grid-template-columns: repeat(2, 1fr); + } + + .forensics-scan-controls { + flex-direction: column; + } + + .forensics-scan-form { + min-width: 100%; + } + + .forensics-scan-actions { + width: 100%; + } + + .forensics-scan-actions button { + flex: 1; + } + + .forensics-result-summary { + flex-direction: column; + align-items: stretch; + } + + .forensics-integrity-badge { + align-self: center; + } + + .metadata-grid { + grid-template-columns: 1fr; + } + + .comparison-controls { + flex-direction: column; + } + + .comparison-select { + min-width: 100%; + } + + .comparison-arrow { + transform: rotate(90deg); + } + + .comparison-header { + flex-direction: column; + align-items: stretch; + text-align: center; + } +} diff --git a/packages/arkham-shard-shell/src/pages/provenance/ProvenancePage.tsx b/packages/arkham-shard-shell/src/pages/provenance/ProvenancePage.tsx index c0754d5..75a514d 100644 --- a/packages/arkham-shard-shell/src/pages/provenance/ProvenancePage.tsx +++ b/packages/arkham-shard-shell/src/pages/provenance/ProvenancePage.tsx @@ -10,6 +10,7 @@ import { Icon } from '../../components/common/Icon'; import { AIAnalystButton } from '../../components/AIAnalyst'; import { useToast } from '../../context/ToastContext'; import { useFetch } from '../../hooks/useFetch'; +import { ForensicsTab } from './ForensicsTab'; import './ProvenancePage.css'; // Types @@ -81,7 +82,7 @@ interface LineageGraph { descendant_count: number; } -type MainTab = 'artifacts' | 'chains' | 'lineage'; +type MainTab = 'artifacts' | 'chains' | 'lineage' | 'forensics'; export function ProvenancePage() { const { toast } = useToast(); @@ -694,6 +695,13 @@ export function ProvenancePage() { Lineage +
{/* Tab Content */} @@ -701,6 +709,7 @@ export function ProvenancePage() { {mainTab === 'artifacts' && renderArtifactsTab()} {mainTab === 'chains' && renderChainsTab()} {mainTab === 'lineage' && renderLineageTab()} + {mainTab === 'forensics' && } {/* Create Chain Dialog */} diff --git a/packages/arkham-shard-shell/src/pages/search/RegexSearchPanel.css b/packages/arkham-shard-shell/src/pages/search/RegexSearchPanel.css new file mode 100644 index 0000000..bc94c57 --- /dev/null +++ b/packages/arkham-shard-shell/src/pages/search/RegexSearchPanel.css @@ -0,0 +1,447 @@ +/* RegexSearchPanel Styles */ + +.regex-panel { + display: flex; + flex-direction: column; + gap: 1rem; + height: 100%; +} + +/* Input Section */ +.regex-input-section { + display: flex; + flex-direction: column; + gap: 0.75rem; + padding: 1rem; + background: var(--surface-secondary); + border: 1px solid var(--border-primary); + border-radius: 0.5rem; +} + +.regex-input-row { + display: flex; + gap: 0.75rem; + align-items: center; +} + +.regex-input-wrapper { + flex: 1; + position: relative; + display: flex; + align-items: center; +} + +.regex-input { + width: 100%; + padding: 0.625rem 2.5rem 0.625rem 0.75rem; + background: var(--surface-primary); + border: 1px solid var(--border-primary); + border-radius: 0.375rem; + color: var(--text-primary); + font-family: monospace; + font-size: 0.875rem; +} + +.regex-input:focus { + outline: none; + border-color: var(--color-primary); +} + +.regex-input::placeholder { + color: var(--text-tertiary); +} + +.regex-input-icon { + position: absolute; + right: 0.75rem; + display: flex; + align-items: center; +} + +/* Validation Feedback */ +.regex-validation { + padding: 0.5rem 0; +} + +.regex-validation-msg { + display: flex; + align-items: center; + gap: 0.5rem; + font-size: 0.8125rem; +} + +.regex-validation-warning { + color: var(--text-tertiary); + font-size: 0.75rem; +} + +/* Performance indicators */ +.regex-perf-fast { + color: #10b981; +} + +.regex-perf-moderate { + color: #f59e0b; +} + +.regex-perf-slow { + color: #f97316; +} + +.regex-perf-dangerous { + color: #ef4444; +} + +.regex-perf-invalid { + color: #ef4444; +} + +/* Flags */ +.regex-flags { + display: flex; + align-items: center; + gap: 0.5rem; +} + +.regex-flags-label { + font-size: 0.8125rem; + color: var(--text-secondary); +} + +.regex-flag-btn { + padding: 0.25rem 0.625rem; + background: var(--surface-primary); + border: 1px solid var(--border-primary); + border-radius: 0.25rem; + color: var(--text-secondary); + font-family: monospace; + font-size: 0.8125rem; + font-weight: 500; + cursor: pointer; + transition: all 0.15s; +} + +.regex-flag-btn:hover { + background: var(--surface-secondary); + border-color: var(--text-tertiary); +} + +.regex-flag-btn.active { + background: var(--color-primary); + border-color: var(--color-primary); + color: white; +} + +/* Presets Section */ +.regex-presets-section { + background: var(--surface-secondary); + border: 1px solid var(--border-primary); + border-radius: 0.5rem; + overflow: hidden; +} + +.regex-presets-toggle { + width: 100%; + display: flex; + justify-content: space-between; + align-items: center; + padding: 0.75rem 1rem; + background: transparent; + border: none; + color: var(--text-primary); + font-size: 0.875rem; + font-weight: 500; + cursor: pointer; + transition: background 0.15s; +} + +.regex-presets-toggle:hover { + background: var(--surface-primary); +} + +.regex-presets-content { + padding: 0.75rem 1rem; + border-top: 1px solid var(--border-primary); + display: flex; + flex-direction: column; + gap: 1rem; +} + +.regex-presets-loading { + padding: 1rem; + text-align: center; + color: var(--text-secondary); + font-size: 0.875rem; +} + +.regex-preset-category { + display: flex; + flex-direction: column; + gap: 0.5rem; +} + +.regex-preset-category-header { + display: flex; + align-items: center; + gap: 0.375rem; + font-size: 0.75rem; + font-weight: 500; + color: var(--text-tertiary); + text-transform: uppercase; + letter-spacing: 0.025em; +} + +.regex-preset-list { + display: flex; + flex-wrap: wrap; + gap: 0.375rem; +} + +.regex-preset-btn { + padding: 0.375rem 0.75rem; + background: var(--surface-primary); + border: 1px solid var(--border-primary); + border-radius: 0.375rem; + color: var(--text-primary); + font-size: 0.8125rem; + cursor: pointer; + transition: all 0.15s; +} + +.regex-preset-btn:hover { + background: var(--color-primary); + border-color: var(--color-primary); + color: white; +} + +/* Results Section */ +.regex-results { + flex: 1; + display: flex; + flex-direction: column; + background: var(--surface-secondary); + border: 1px solid var(--border-primary); + border-radius: 0.5rem; + overflow: hidden; + min-height: 200px; +} + +.regex-results-loading { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: 0.75rem; + padding: 2rem; + color: var(--text-secondary); +} + +.regex-results-error { + display: flex; + align-items: center; + gap: 0.5rem; + padding: 1rem; + color: #ef4444; + font-size: 0.875rem; +} + +.regex-results-header { + display: flex; + justify-content: space-between; + align-items: center; + padding: 0.75rem 1rem; + background: var(--surface-primary); + border-bottom: 1px solid var(--border-primary); + font-size: 0.875rem; + color: var(--text-secondary); +} + +.regex-results-header strong { + color: var(--text-primary); +} + +.regex-results-empty { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: 0.5rem; + padding: 2rem; + color: var(--text-tertiary); + text-align: center; +} + +.regex-results-empty p { + margin: 0; + font-size: 0.875rem; +} + +.regex-results-hint { + font-size: 0.75rem !important; + color: var(--text-tertiary); +} + +/* Match List */ +.regex-match-list { + flex: 1; + overflow-y: auto; +} + +.regex-match-item { + display: flex; + gap: 0.75rem; + padding: 0.75rem 1rem; + border-bottom: 1px solid var(--border-primary); + cursor: pointer; + transition: background 0.15s; +} + +.regex-match-item:last-child { + border-bottom: none; +} + +.regex-match-item:hover { + background: var(--surface-primary); +} + +.regex-match-item > svg { + flex-shrink: 0; + color: var(--text-tertiary); + margin-top: 0.125rem; +} + +.regex-match-content { + flex: 1; + min-width: 0; +} + +.regex-match-header { + display: flex; + align-items: center; + gap: 0.5rem; + flex-wrap: wrap; + margin-bottom: 0.375rem; +} + +.regex-match-title { + font-size: 0.875rem; + font-weight: 500; + color: var(--text-primary); +} + +.regex-match-badge { + padding: 0.125rem 0.375rem; + background: var(--surface-secondary); + border-radius: 0.25rem; + font-size: 0.6875rem; + color: var(--text-tertiary); +} + +.regex-match-context { + font-family: monospace; + font-size: 0.8125rem; + color: var(--text-secondary); + line-height: 1.5; + word-break: break-word; +} + +.regex-context-text { + color: var(--text-secondary); +} + +.regex-match-highlight { + background: rgba(59, 130, 246, 0.3); + color: #60a5fa; + padding: 0.0625rem 0.125rem; + border-radius: 0.125rem; +} + +/* Spinner animation */ +.spin { + animation: spin 1s linear infinite; +} + +@keyframes spin { + from { + transform: rotate(0deg); + } + to { + transform: rotate(360deg); + } +} + +/* Button styles - reusing common patterns */ +.btn { + display: inline-flex; + align-items: center; + gap: 0.375rem; + padding: 0.5rem 1rem; + border-radius: 0.375rem; + font-size: 0.875rem; + font-weight: 500; + cursor: pointer; + transition: all 0.15s; +} + +.btn:disabled { + opacity: 0.5; + cursor: not-allowed; +} + +.btn-primary { + background: var(--color-primary); + border: none; + color: white; +} + +.btn-primary:hover:not(:disabled) { + background: var(--color-primary-hover, #2563eb); +} + +/* Detected Patterns Section */ +.regex-detected-badge { + padding: 0.125rem 0.5rem; + background: var(--color-primary); + border-radius: 0.75rem; + font-size: 0.75rem; + font-weight: 500; + color: white; + margin-left: auto; + margin-right: 0.5rem; +} + +.regex-detected-empty { + display: flex; + align-items: center; + gap: 0.5rem; + padding: 0.75rem; + color: var(--text-tertiary); + font-size: 0.8125rem; +} + +.regex-detected-count { + margin-left: 0.375rem; + padding: 0.0625rem 0.375rem; + background: rgba(255, 255, 255, 0.15); + border-radius: 0.25rem; + font-size: 0.6875rem; +} + +/* Status indicators for detected patterns */ +.regex-detected-status-detected { + border-left: 2px solid #f59e0b; +} + +.regex-detected-status-confirmed { + border-left: 2px solid #10b981; +} + +.regex-detected-status-dismissed { + opacity: 0.5; + border-left: 2px solid var(--border-primary); +} + +.regex-detected-status-archived { + opacity: 0.6; + border-left: 2px solid var(--text-tertiary); +} diff --git a/packages/arkham-shard-shell/src/pages/search/RegexSearchPanel.tsx b/packages/arkham-shard-shell/src/pages/search/RegexSearchPanel.tsx new file mode 100644 index 0000000..2eb5107 --- /dev/null +++ b/packages/arkham-shard-shell/src/pages/search/RegexSearchPanel.tsx @@ -0,0 +1,398 @@ +/** + * Regex Search Panel + * + * A dedicated panel for regex pattern searching with preset support. + */ + +import React, { useState, useEffect, useCallback } from 'react'; +import { Icon } from '../../components/common/Icon'; +import { useRegexSearch, useRegexPresets, useDetectedPatterns, validatePattern } from './api'; +import type { RegexPreset, RegexMatch, PatternValidation, DetectedPattern } from './types'; +import './RegexSearchPanel.css'; + +interface RegexSearchPanelProps { + projectId?: string; + onMatchClick?: (match: RegexMatch) => void; +} + +const CATEGORY_ICONS: Record = { + contact: 'Mail', + pii: 'Shield', + financial: 'DollarSign', + technical: 'Globe', + temporal: 'Calendar', + custom: 'Hash', +}; + +const PATTERN_TYPE_ICONS: Record = { + recurring_theme: 'Repeat', + behavioral: 'Activity', + temporal: 'Clock', + correlation: 'Link', + linguistic: 'Type', + structural: 'Layout', + custom: 'Settings', +}; + +export default function RegexSearchPanel({ projectId, onMatchClick }: RegexSearchPanelProps) { + const [pattern, setPattern] = useState(''); + const [searchPattern, setSearchPattern] = useState(''); + const [flags, setFlags] = useState([]); + const [showPresets, setShowPresets] = useState(true); + const [showDetected, setShowDetected] = useState(true); + const [validation, setValidation] = useState(null); + const [validating, setValidating] = useState(false); + + const { data: presets, loading: presetsLoading } = useRegexPresets(); + const { data: detectedPatterns, loading: detectedLoading } = useDetectedPatterns({ limit: 50 }); + const { + data: searchResults, + loading: searchLoading, + error: searchError, + } = useRegexSearch({ + pattern: searchPattern, + request: { flags, project_id: projectId, limit: 100 }, + enabled: !!searchPattern, // Allow search even without validation - backend validates + }); + + // Validate pattern on change (debounced) + useEffect(() => { + if (!pattern) { + setValidation(null); + return; + } + + const timer = setTimeout(async () => { + setValidating(true); + try { + const result = await validatePattern(pattern); + setValidation(result); + } catch { + setValidation({ valid: false, error: 'Validation failed', estimated_performance: 'invalid' }); + } finally { + setValidating(false); + } + }, 300); + + return () => clearTimeout(timer); + }, [pattern]); + + const handleSearch = useCallback(() => { + // Allow search even if validation hasn't completed yet - backend will validate + if (pattern.trim()) { + setSearchPattern(pattern); + } + }, [pattern]); + + const handlePresetClick = useCallback((preset: RegexPreset) => { + setPattern(preset.pattern); + setFlags(preset.flags || []); + }, []); + + const handleDetectedPatternClick = useCallback((detected: DetectedPattern) => { + // Use the first regex pattern from criteria if available + if (detected.criteria?.regex_patterns?.length) { + setPattern(detected.criteria.regex_patterns[0]); + setFlags([]); + } else if (detected.criteria?.keywords?.length) { + // Fall back to keyword as a simple pattern + setPattern(detected.criteria.keywords.join('|')); + setFlags(['case_insensitive']); + } + }, []); + + const handleKeyDown = useCallback( + (e: React.KeyboardEvent) => { + if (e.key === 'Enter' && pattern.trim()) { + handleSearch(); + } + }, + [handleSearch, pattern] + ); + + const toggleFlag = useCallback((flag: string) => { + setFlags((prev) => (prev.includes(flag) ? prev.filter((f) => f !== flag) : [...prev, flag])); + }, []); + + // Group presets by category + const presetsByCategory = presets.reduce((acc, preset) => { + const cat = preset.category || 'custom'; + if (!acc[cat]) acc[cat] = []; + acc[cat].push(preset); + return acc; + }, {} as Record); + + // Filter and group detected patterns that have searchable criteria + const searchableDetectedPatterns = detectedPatterns.filter( + (p) => p.criteria?.regex_patterns?.length || p.criteria?.keywords?.length + ); + + // Group detected patterns by type + const detectedByType = searchableDetectedPatterns.reduce((acc, pattern) => { + const type = pattern.pattern_type || 'custom'; + if (!acc[type]) acc[type] = []; + acc[type].push(pattern); + return acc; + }, {} as Record); + + const getPerformanceClass = (perf: string) => { + switch (perf) { + case 'fast': return 'regex-perf-fast'; + case 'moderate': return 'regex-perf-moderate'; + case 'slow': return 'regex-perf-slow'; + case 'dangerous': return 'regex-perf-dangerous'; + default: return 'regex-perf-invalid'; + } + }; + + return ( +
+ {/* Pattern Input */} +
+
+
+ setPattern(e.target.value)} + onKeyDown={handleKeyDown} + placeholder="Enter regex pattern (e.g., \d{3}-\d{4})" + className="regex-input" + /> + {validating && ( +
+ +
+ )} + {!validating && validation && ( +
+ {validation.valid ? ( + + ) : ( + + )} +
+ )} +
+ +
+ + {/* Validation Feedback */} + {validation && ( +
+ {validation.valid ? ( +
+ + Performance: {validation.estimated_performance} + {validation.estimated_performance === 'dangerous' && ( + + (may cause slowdowns on large datasets) + + )} +
+ ) : ( +
+ + {validation.error} +
+ )} +
+ )} + + {/* Regex Flags */} +
+ Flags: + {[ + { id: 'case_insensitive', label: 'Case Insensitive', short: 'i' }, + { id: 'multiline', label: 'Multiline', short: 'm' }, + { id: 'dotall', label: 'Dot All', short: 's' }, + ].map((flag) => ( + + ))} +
+
+ + {/* Presets Section */} +
+ + + {showPresets && ( +
+ {presetsLoading ? ( +
Loading presets...
+ ) : ( + Object.entries(presetsByCategory).map(([category, categoryPresets]) => ( +
+
+ + {category} +
+
+ {categoryPresets.map((preset) => ( + + ))} +
+
+ )) + )} +
+ )} +
+ + {/* Detected Patterns from Ingest Section */} +
+ + + {showDetected && ( +
+ {detectedLoading ? ( +
Loading detected patterns...
+ ) : searchableDetectedPatterns.length === 0 ? ( +
+ + No patterns detected yet. Process documents to auto-detect patterns. +
+ ) : ( + Object.entries(detectedByType).map(([type, typePatterns]) => ( +
+
+ + {type.replace(/_/g, ' ')} +
+
+ {typePatterns.map((detected) => ( + + ))} +
+
+ )) + )} +
+ )} +
+ + {/* Results Section */} +
+ {searchLoading && ( +
+ + Searching... +
+ )} + + {searchError && ( +
+ + {searchError.message} +
+ )} + + {searchResults && !searchLoading && ( + <> + {/* Results Header */} +
+ + Found {searchResults.total_matches} matches + in {searchResults.documents_searched} documents + + {searchResults.duration_ms.toFixed(1)}ms +
+ + {/* Match List */} +
+ {searchResults.matches.length === 0 ? ( +
+ +

No matches found for this pattern

+
+ ) : ( + searchResults.matches.map((match, index) => ( +
onMatchClick?.(match)} + > + +
+
+ {match.document_title} + {match.page_number && ( + Page {match.page_number} + )} + Line {match.line_number} +
+
+ {match.context.split(match.match_text).map((part, i, arr) => ( + + {part} + {i < arr.length - 1 && ( + {match.match_text} + )} + + ))} +
+
+
+ )) + )} +
+ + )} + + {!searchResults && !searchLoading && !searchError && ( +
+ +

Enter a regex pattern and click Search

+

Or select a preset pattern above

+
+ )} +
+
+ ); +} diff --git a/packages/arkham-shard-shell/src/pages/search/SearchPage.tsx b/packages/arkham-shard-shell/src/pages/search/SearchPage.tsx index 17fa5e0..f39bf3e 100644 --- a/packages/arkham-shard-shell/src/pages/search/SearchPage.tsx +++ b/packages/arkham-shard-shell/src/pages/search/SearchPage.tsx @@ -11,8 +11,9 @@ import { AIAnalystButton } from '../../components/AIAnalyst'; import { useToast } from '../../context/ToastContext'; import { SearchResultCard } from './SearchResultCard'; import { RAGChatPanel } from './RAGChatPanel'; +import RegexSearchPanel from './RegexSearchPanel'; import { useSearch, findSimilar } from './api'; -import type { SearchMode, SearchFilters, SearchResultItem } from './types'; +import type { SearchMode, SearchFilters, SearchResultItem, RegexMatch } from './types'; export function SearchPage() { const { toast } = useToast(); @@ -215,6 +216,13 @@ export function SearchPage() { Keyword + {/* Filters Toggle */} @@ -344,8 +352,20 @@ export function SearchPage() { )} - {/* Results Section */} - {queryFromUrl && ( + {/* Regex Search Panel - Shown when regex mode is selected */} + {searchMode === 'regex' && ( +
+ { + navigate(`/documents/${match.document_id}`); + }} + /> +
+ )} + + {/* Results Section - Shown for non-regex modes */} + {searchMode !== 'regex' && queryFromUrl && (
{loading && (
diff --git a/packages/arkham-shard-shell/src/pages/search/api.ts b/packages/arkham-shard-shell/src/pages/search/api.ts index 8d057da..3a35276 100644 --- a/packages/arkham-shard-shell/src/pages/search/api.ts +++ b/packages/arkham-shard-shell/src/pages/search/api.ts @@ -11,6 +11,13 @@ import type { Suggestion, SimilarResponse, AvailableFilters, + RegexPreset, + RegexSearchRequest, + RegexSearchResponse, + PatternValidation, + PatternExtraction, + ExtractionStats, + DetectedPattern, } from './types'; const API_PREFIX = '/api/search'; @@ -107,6 +114,112 @@ export async function getAvailableFilters(query?: string): Promise { + return fetchAPI('/regex', { + method: 'POST', + body: JSON.stringify(request), + }); +} + +export async function validatePattern(pattern: string): Promise { + return fetchAPI('/regex/validate', { + method: 'POST', + body: JSON.stringify({ pattern }), + }); +} + +export async function getRegexPresets(category?: string): Promise { + const params = new URLSearchParams(); + if (category) params.set('category', category); + return fetchAPI(`/regex/presets?${params.toString()}`); +} + +export async function createRegexPreset(preset: { + name: string; + pattern: string; + description?: string; + category?: string; + flags?: string[]; +}): Promise { + return fetchAPI('/regex/presets', { + method: 'POST', + body: JSON.stringify(preset), + }); +} + +export async function deleteRegexPreset(presetId: string): Promise<{ success: boolean }> { + return fetchAPI<{ success: boolean }>(`/regex/presets/${presetId}`, { + method: 'DELETE', + }); +} + +export async function getPatternExtractions(params: { + document_id?: string; + preset_id?: string; + category?: string; + limit?: number; + offset?: number; +}): Promise<{ extractions: PatternExtraction[]; total: number }> { + const searchParams = new URLSearchParams(); + if (params.document_id) searchParams.set('document_id', params.document_id); + if (params.preset_id) searchParams.set('preset_id', params.preset_id); + if (params.category) searchParams.set('category', params.category); + if (params.limit) searchParams.set('limit', params.limit.toString()); + if (params.offset) searchParams.set('offset', params.offset.toString()); + return fetchAPI<{ extractions: PatternExtraction[]; total: number }>( + `/regex/extractions?${searchParams.toString()}` + ); +} + +export async function getExtractionStats(): Promise { + return fetchAPI('/regex/extractions/stats'); +} + +export async function triggerPatternExtraction( + documentId: string, + presetIds?: string[] +): Promise<{ document_id: string; extractions: number; presets_used: string[] }> { + const body = presetIds ? JSON.stringify({ preset_ids: presetIds }) : '{}'; + return fetchAPI<{ document_id: string; extractions: number; presets_used: string[] }>( + `/regex/extract/${documentId}`, + { + method: 'POST', + body, + } + ); +} + +// ============================================ +// Detected Patterns from Patterns Shard +// ============================================ + +export async function getDetectedPatterns(params?: { + status?: string; + pattern_type?: string; + has_regex?: boolean; + limit?: number; +}): Promise<{ items: DetectedPattern[]; total: number }> { + const searchParams = new URLSearchParams(); + if (params?.status) searchParams.set('status', params.status); + if (params?.pattern_type) searchParams.set('type', params.pattern_type); + if (params?.limit) searchParams.set('page_size', params.limit.toString()); + + const response = await fetch(`/api/patterns/?${searchParams.toString()}`, { + headers: { 'Content-Type': 'application/json' }, + }); + + if (!response.ok) { + const error = await response.json().catch(() => ({ detail: response.statusText })); + throw new Error(error.detail || `HTTP ${response.status}`); + } + + return response.json(); +} + // ============================================ // React Hooks // ============================================ @@ -251,6 +364,117 @@ export function useKeyword(options: UseSearchOptions): UseSearchResult { return { data, loading, error, refetch }; } +// ============================================ +// Regex Search Hooks +// ============================================ + +interface UseRegexSearchOptions { + pattern: string; + request?: Partial; + enabled?: boolean; +} + +interface UseRegexSearchResult { + data: RegexSearchResponse | null; + loading: boolean; + error: Error | null; + refetch: () => void; +} + +export function useRegexSearch(options: UseRegexSearchOptions): UseRegexSearchResult { + const [data, setData] = useState(null); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + + const requestJson = JSON.stringify(options.request || {}); + + const fetchData = useCallback(async () => { + if (!options.pattern || options.enabled === false) { + setLoading(false); + return; + } + + setLoading(true); + setError(null); + + try { + const request = JSON.parse(requestJson); + const result = await performRegexSearch({ + pattern: options.pattern, + ...request, + }); + setData(result); + setError(null); + } catch (err) { + setError(err instanceof Error ? err : new Error('Unknown error')); + } finally { + setLoading(false); + } + }, [options.pattern, requestJson, options.enabled]); + + useEffect(() => { + fetchData(); + }, [fetchData]); + + const refetch = useCallback(() => { + fetchData(); + }, [fetchData]); + + return { data, loading, error, refetch }; +} + +export function useRegexPresets(category?: string) { + const [data, setData] = useState([]); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + + const fetchData = useCallback(async () => { + setLoading(true); + setError(null); + + try { + const result = await getRegexPresets(category); + setData(result); + } catch (err) { + setError(err instanceof Error ? err : new Error('Unknown error')); + } finally { + setLoading(false); + } + }, [category]); + + useEffect(() => { + fetchData(); + }, [fetchData]); + + return { data, loading, error, refetch: fetchData }; +} + +export function useExtractionStats() { + const [data, setData] = useState(null); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + + const fetchData = useCallback(async () => { + setLoading(true); + setError(null); + + try { + const result = await getExtractionStats(); + setData(result); + } catch (err) { + setError(err instanceof Error ? err : new Error('Unknown error')); + } finally { + setLoading(false); + } + }, []); + + useEffect(() => { + fetchData(); + }, [fetchData]); + + return { data, loading, error, refetch: fetchData }; +} + interface UseSimilarResult { data: SimilarResponse | null; loading: boolean; @@ -293,3 +517,38 @@ export function useSimilar(docId: string | null, limit = 10): UseSimilarResult { return { data, loading, error, refetch }; } + +export function useDetectedPatterns(params?: { + status?: string; + pattern_type?: string; + has_regex?: boolean; + limit?: number; +}) { + const [data, setData] = useState([]); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + + const paramsJson = JSON.stringify(params || {}); + + const fetchData = useCallback(async () => { + setLoading(true); + setError(null); + + try { + const parsedParams = JSON.parse(paramsJson); + const result = await getDetectedPatterns(parsedParams); + setData(result.items || []); + } catch (err) { + setError(err instanceof Error ? err : new Error('Unknown error')); + setData([]); + } finally { + setLoading(false); + } + }, [paramsJson]); + + useEffect(() => { + fetchData(); + }, [fetchData]); + + return { data, loading, error, refetch: fetchData }; +} diff --git a/packages/arkham-shard-shell/src/pages/search/types.ts b/packages/arkham-shard-shell/src/pages/search/types.ts index d578e98..ebe0827 100644 --- a/packages/arkham-shard-shell/src/pages/search/types.ts +++ b/packages/arkham-shard-shell/src/pages/search/types.ts @@ -5,7 +5,7 @@ */ // Search modes -export type SearchMode = 'hybrid' | 'semantic' | 'keyword'; +export type SearchMode = 'hybrid' | 'semantic' | 'keyword' | 'regex'; // Sort options export type SortBy = 'relevance' | 'date' | 'title'; @@ -83,3 +83,96 @@ export interface AvailableFilters { projects?: { value: string; label: string; count: number }[]; date_range?: { min: string; max: string }; } + +// Regex search types +export interface RegexPreset { + id: string; + name: string; + pattern: string; + description: string; + category: string; + flags: string[]; + is_system: boolean; +} + +export interface RegexMatch { + document_id: string; + document_title: string; + page_number: number | null; + chunk_id: string; + match_text: string; + context: string; + start_offset: number; + end_offset: number; + line_number: number; +} + +export interface RegexSearchRequest { + pattern: string; + flags?: string[]; + project_id?: string; + document_ids?: string[]; + limit?: number; + offset?: number; + context_chars?: number; +} + +export interface RegexSearchResponse { + pattern: string; + matches: RegexMatch[]; + total_matches: number; + total_chunks_with_matches: number; + documents_searched: number; + duration_ms: number; + error: string | null; +} + +export interface PatternValidation { + valid: boolean; + error: string | null; + estimated_performance: 'fast' | 'moderate' | 'slow' | 'dangerous' | 'invalid'; +} + +export interface PatternExtraction { + id: string; + document_id: string; + preset_id: string; + preset_name: string | null; + category: string | null; + match_text: string; + context: string | null; + page_number: number | null; + chunk_id: string | null; + start_offset: number | null; + end_offset: number | null; + line_number: number | null; + extracted_at: string | null; +} + +export interface ExtractionStats { + total_extractions: number; + documents_with_patterns: number; + by_category: Record; + by_preset: Record; +} + +// Detected pattern from patterns shard (auto-extracted during ingest) +export interface DetectedPattern { + id: string; + name: string; + description: string | null; + pattern_type: string; + status: 'detected' | 'confirmed' | 'dismissed' | 'archived'; + confidence: number; + match_count: number; + document_count: number; + first_detected: string | null; + last_matched: string | null; + detection_method: string; + criteria: { + keywords?: string[]; + regex_patterns?: string[]; + entity_types?: string[]; + min_occurrences?: number; + } | null; +}