diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml new file mode 100644 index 0000000..d2f2c32 --- /dev/null +++ b/.github/workflows/release-gate.yml @@ -0,0 +1,55 @@ +name: Release gate + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +concurrency: + group: release-gate-${{ github.ref }} + cancel-in-progress: true + +jobs: + release-gate: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + cache: pip + + - name: Install locked runtime dependencies + run: python -m pip install --disable-pip-version-check -r requirements.txt + + - name: Install pinned release tools + run: python -m pip install --disable-pip-version-check pip-audit==2.10.1 bandit==1.9.4 ruff==0.16.3 + + - name: Audit runtime dependencies + run: python -m pip_audit -r requirements.txt + + - name: Run tests + run: python -W error::ResourceWarning -m unittest discover -s tests -v + + - name: Compile Python sources + run: python -m py_compile run_osintai.py src/osintai/*.py tests/*.py + + - name: Run security analysis + run: bandit -q -r src run_osintai.py -ll -ii + + - name: Check correctness lint rules + run: ruff check --select E9,F63,F7,F82 src tests run_osintai.py + + - name: Verify release identity + run: | + python run_osintai.py --version | grep -Fx "OSINTai 4.0.0" + python run_osintai.py --help >/dev/null diff --git a/.gitignore b/.gitignore index 488721b..dabc8e7 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,8 @@ id_ed25519 data/runs/ data/cache/ cases/ +/OSINTAI_SYNTHESIS/ +/TODO OSINTai_FILES1/ proxies/ proxy*.txt diff --git a/README.md b/README.md index 1bf0793..0358585 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ OSINTai Logo -# OSINTai v3.4 - Advanced AI-Powered OSINT Web Crawler +# OSINTai v4 - Advanced Local-First OSINT Web Crawler [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/) @@ -11,6 +11,17 @@ **OSINTai** is a cutting-edge, AI-enhanced web crawler engineered for **Open Source Intelligence (OSINT)** professionals, cybersecurity researchers, and digital investigators. Leveraging advanced asynchronous processing, intelligent proxy rotation, and state-of-the-art LLM analysis, OSINTai automates comprehensive intelligence gathering from web sources with unparalleled efficiency and accuracy. +## v4 Highlights + +- Deterministic post-crawl analysis with evidence-labelled findings, correlations, timelines, hypotheses, and leads +- Unicode and homoglyph detection, sensitive-infrastructure checks, secret-presence reporting, and generated-text fingerprints +- Cross-source entity correlation with candidate-only relationships and supporting URLs +- Optional run-level model analysis and multi-model cross-checks that preserve disagreement +- Deterministic model-output evaluation and inert local training-dataset exports +- Multi-seed scope enforcement across queueing and every redirect hop +- HTML/XHTML-only response handling with configurable decoded-size limits +- Loopback-only Ollama communication, validated run identifiers, and atomic JSON checkpoints + **Key Capabilities:** - **High-Performance Async Crawling** with intelligent concurrency controls - **AI-Powered Content Analysis** using Ollama LLMs for structured intelligence extraction @@ -101,7 +112,7 @@ Pages are automatically ranked by total score, with AI-enhanced intelligence rec ```bash # 1. Clone the repository -git clone https://github.com/yourusername/OSINTai.git +git clone https://github.com/gs-ai/OSINTai.git cd OSINTai # 2. Create conda environment @@ -218,9 +229,18 @@ https://site3.com Then run: ```bash -python run_osintai.py --seed "https://example.com" # Will also check for seed_urls.txt +python run_osintai.py \ + --seed-file seed_urls.txt \ + --depth 2 \ + --max 150 \ + --same-domain ``` +With multiple seeds, `--same-domain` permits links on any listed seed hostname while +rejecting links and redirects to other hosts. Responses are streamed and accepted only +when they are HTML/XHTML and no larger than 2,000,000 decoded bytes. Change the bound +with `--max-response-bytes` if a specific authorized page requires it. + --- ## Command Line Options @@ -233,7 +253,7 @@ usage: run_osintai.py [-h] [--seed SEED] [--depth DEPTH] [--max MAX] [--no-ollama] [--hunt HUNT] [--hunt-max HUNT_MAX] [--run-id RUN_ID] -OSINTai v3.4 FULL (async + proxy + dedupe + embeddings + hunt + graph export) +OSINTai v4 (async crawling and analysis) required arguments: --seed SEED Seed URL (or use seed_urls.txt file) @@ -252,6 +272,76 @@ optional arguments: --hunt HUNT Comma-separated hunt terms (optional) --hunt-max HUNT_MAX Max lead URLs per page from hunt mode (default: 50) --run-id RUN_ID Optional run ID override (default: auto-generated) + +analysis options: + --profile {default,survey} Named run profile applied before explicit flags + --prompt-profile {standard,threat} Page analysis lens (default: standard) + --no-analysis Skip the post-crawl analysis stage (crawl and report only) + --gap-days GAP_DAYS Temporal gap threshold in days (default: 90) + --leads-per-kind N Max entities of each kind expanded into leads (default: 10) + +optional deeper analysis (off by default): + --deep Model-assisted run-level analysis pass + --cross-check MODELS Comma-separated extra Ollama models to cross-check claims + --evaluate Score analysis quality against the investigative rubric + --training-export Export local training/evaluation datasets (needs --evaluate) + --experimental-recursive N Extra bounded refinement passes, 0-3 (default: 0) +``` + +### Analysis Layer + +After the crawl completes, OSINTai runs a deterministic analysis stage over what the crawl +collected. It is fast, works fully offline, and is on by default; `--no-analysis` skips it. + +**Deterministic checks (no model involved):** +- Unicode / homoglyph detection on collected domains and handles, including zero-width characters +- Sensitive infrastructure: `.gov` / `.mil` identifiers and non-routable (RFC1918) IP exposure +- Credential and secret exposure, including JWT claim decoding. Matched secret values are + never written into findings or reports, only their presence and location +- Generated-content fingerprinting, flagging pages carrying unedited language-model output +- Cross-source correlation producing scored candidate links backed by evidence URLs +- Temporal analysis: ordered timeline, activity gaps, open trailing gaps, bursts +- Recurring cross-page signals and score outliers + +**Observed versus derived.** Every analytical statement is labelled by origin: + +| Label | Meaning | +| --- | --- | +| `OBSERVED` | Present in the fetched source material | +| `DERIVED` | Computed deterministically from observed material. Reproducible | +| `MODEL` | A language model's interpretation. Not a measurement | +| `HYPOTHESIS` | A proposed explanation, with what would confirm and refute it | + +Confidence is reported by kind (`source_support`, `deterministic`, `model_self`, +`cross_model`, `human_review`) and the kinds are never merged. A model's self-reported +confidence is recorded, labelled, and never treated as factual reliability. + +**Optional deeper modes.** Nothing below runs unless requested, and the default path stays +one model call per page: + +```bash +# Run-level model analysis across the whole crawl +python run_osintai.py --seed "https://target.example/" --deep + +# Cross-check the model's risk assessments against a second model. +# Where models disagree, the disagreement is preserved, not averaged away. +python run_osintai.py --seed "https://target.example/" --cross-check "mistral:7b-instruct-v0.3-q5_K_M" + +# Score analysis quality and export portable local training/evaluation datasets +python run_osintai.py --seed "https://target.example/" --evaluate --training-export +``` + +**Training export.** `--training-export` writes evaluation tasks, scored analyses, and +preference records into the local run directory. OSINTai performs no training, includes no +training dependencies, and never modifies model weights. + +### Run Profiles + +`--profile survey` applies a deeper, narrower, politer parameter set (depth 3, max 300, +same-domain, concurrency 10, per-host 3). Explicit flags always override the profile. + +```bash +python run_osintai.py --seed "https://target.example/" --profile survey --run-id "target.example_001" ``` --- @@ -277,6 +367,17 @@ Each crawl generates a timestamped directory under `data/runs/` with comprehensi - **`graph_nodes.jsonl`** - Graph nodes for network visualization and analysis - **`graph_edges.jsonl`** - Graph relationships and connections +### Analysis Results (unless `--no-analysis`) +- **`analysis_report.txt`** - Findings separated by origin, correlations, timeline, hypotheses, leads +- **`findings.jsonl`** - Every finding with priority, evidence, sources, confidence, and next step +- **`correlations.jsonl`** - Scored candidate entity links with the evidence URLs behind each +- **`timeline.jsonl`** - Ordered event stream with original timestamps preserved +- **`hypotheses.jsonl`** - Labelled hypotheses with confirm/refute conditions +- **`leads.jsonl`** - Pivot lookup paths per identifier, with false-positive risk notes +- **`analysis_summary.json`** - Per-stage notes, errors, and statistics +- **`run_manifest.json`** - What was requested, which models answered, and where output went +- **`training_export/`** - Local dataset package (only with `--evaluate --training-export`) + ### Raw Content Archives - **`pages_raw/`** - Original HTML content for forensic analysis - **`pages_text/`** - Extracted text content for processing and review @@ -531,13 +632,14 @@ src/osintai/ ### Operational Security Features - **Anonymization**: Proxy rotation and user-agent randomization -- **Stealth Techniques**: Adaptive delays and request patterns -- **Data Sanitization**: No sensitive information logging +- **Bounded Requests**: Adaptive delays, concurrency limits, scoped redirects, and response-size limits +- **Sensitive Findings**: Secret-shaped values are not repeated in generated findings or reports +- **Local Data**: Raw pages and run artifacts stay under ignored `data/runs/`; operators must protect that directory - **Provenance Tracking**: Complete audit trail for intelligence chain of custody ### Ethical Usage Guidelines - **Legal Compliance**: Authorized access to publicly available information only -- **Terms Respect**: Honor site policies, robots.txt, and service agreements +- **Terms Respect**: Configure and operate crawls in accordance with site policies and service agreements - **Data Handling**: Secure storage and responsible intelligence dissemination - **Attribution**: Maintain source credibility and investigation integrity @@ -554,7 +656,7 @@ src/osintai/ ### Development Environment ```bash # Fork and clone -git clone https://github.com/yourusername/OSINTai.git +git clone https://github.com/gs-ai/OSINTai.git cd OSINTai # Create development environment @@ -577,7 +679,7 @@ pip install black flake8 pytest mypy 1. Fork the repository 2. Create a feature branch (`git checkout -b feature/amazing-enhancement`) 3. Implement changes with tests -4. Ensure all tests pass (`pytest`) +4. Ensure all tests pass (`python -m unittest discover -s tests -v`) 5. Format code (`black .`) 6. Lint code (`flake8`) 7. Commit with clear messages @@ -603,7 +705,15 @@ pip install black flake8 pytest mypy ## Changelog -### v3.4 FULL (2026-07-11) - Current Release +### v4.0.0 (2026-08-14) - Current Release +- **Evidence-Labelled Analysis**: Findings distinguish observed, derived, model-assisted, and hypothetical statements +- **Expanded Deterministic Checks**: Homoglyphs, sensitive infrastructure, secret presence, generated text, temporal gaps, and outliers +- **Cross-Source Intelligence**: Entity normalization, evidence-backed candidate correlations, timelines, hypotheses, and pivot leads +- **Optional Deep Analysis**: Bounded run-level analysis, multi-model cross-checks, deterministic evaluation, and local dataset export +- **Network Guardrails**: Redirects are scope-checked before each request; non-HTML and oversized responses are rejected +- **Runtime Hardening**: Ollama is loopback-only, run IDs cannot escape the output directory, and checkpoints are atomic + +### v3.4 FULL (2026-07-11) - **Efficient Crawl Scheduling**: O(1) queue operations with duplicate suppression across queued, active, and visited URLs - **Robust Resume State**: Restores pending work, visited URLs, and Simhash history without losing multi-seed context - **Active Proxy Rotation**: Proxies are applied to outbound requests with per-proxy success/failure scoring and persisted health state @@ -630,8 +740,8 @@ pip install black flake8 pytest mypy ## Support & Community -- **Bug Reports**: [GitHub Issues](https://github.com/yourusername/OSINTai/issues) -- **Feature Requests**: [GitHub Discussions](https://github.com/yourusername/OSINTai/discussions) +- **Bug Reports**: [GitHub Issues](https://github.com/gs-ai/OSINTai/issues) +- **Feature Requests**: [GitHub Discussions](https://github.com/gs-ai/OSINTai/discussions) - **Documentation**: Comprehensive in-code docstrings and this README - **Community**: OSINT professional forums and security research communities @@ -645,4 +755,4 @@ Built for the OSINT community with contributions from security researchers, digi --- -*OSINTai v3.4 - Illuminating the shadows of open source intelligence.* +*OSINTai v4 - Illuminating the shadows of open source intelligence.* diff --git a/requirements.txt b/requirements.txt index 4769d53..cc6740a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -beautifulsoup4>=4.12.2 -lxml>=5.0.0 -httpx>=0.27.0 -anyio>=4.0.0 +beautifulsoup4==4.15.0 +lxml==6.1.1 +httpx==0.28.1 +anyio==4.14.2 diff --git a/src/osintai/__init__.py b/src/osintai/__init__.py index c4cc7b0..674433c 100644 --- a/src/osintai/__init__.py +++ b/src/osintai/__init__.py @@ -1,3 +1,5 @@ +__version__ = "4.0.0" + __all__ = [ "cli", "crawler", @@ -12,5 +14,18 @@ "dedupe", "hunt", "graph_export", - "ollama_api" + "ollama_api", + # Analysis layer + "provenance", + "entities", + "patterns", + "correlation", + "temporal", + "pivots", + "hypotheses", + "prompts", + "multimodel", + "evaluation", + "training_export", + "pipeline" ] diff --git a/src/osintai/cli.py b/src/osintai/cli.py index 1457869..182e6c1 100644 --- a/src/osintai/cli.py +++ b/src/osintai/cli.py @@ -1,15 +1,38 @@ import os +import sys import argparse import json import asyncio +import re +import time +from urllib.parse import urlparse -from osintai.storage import safe_mkdir, now_run_id, load_lines +from osintai.storage import safe_mkdir, now_run_id, load_lines, write_json from osintai.extractor import Extractor from osintai.proxy_pool import ProxyPool from osintai.fetcher import AsyncFetcher from osintai.crawler import AsyncCrawler from osintai.scoring import rank_pages -from osintai.report import write_report +from osintai.report import write_report, write_analysis_report +from osintai.ollama_api import OllamaAPI +from osintai.pipeline import AnalysisOptions, analyze_run +from osintai.prompts import PAGE_PROFILES, STANDARD +from osintai import __version__ + +RUN_ID_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") + +# Named run profiles. `survey` encodes the deeper, narrower, politer parameters from the +# OSINTai run template: go further into one target and go easier on its server. +RUN_PROFILES = { + "default": {}, + "survey": { + "depth": 3, + "max": 300, + "same_domain": True, + "concurrency": 10, + "per_host": 3, + }, +} def _read_jsonl(path: str): if not os.path.exists(path): @@ -22,19 +45,103 @@ def _read_jsonl(path: str): continue try: rows.append(json.loads(line)) - except: - pass + except (json.JSONDecodeError, TypeError): + continue return rows + +def _valid_seed_url(value: str) -> bool: + try: + parsed = urlparse(value) + return ( + parsed.scheme in {"http", "https"} + and bool(parsed.hostname) + and parsed.username is None + and parsed.password is None + ) + except ValueError: + return False + + +def _load_seed_file(path: str) -> list[str]: + """Load and validate HTTP(S) seed URLs, preserving their file order.""" + seeds: list[str] = [] + with open(path, "r", encoding="utf-8") as handle: + for line_number, raw_line in enumerate(handle, start=1): + value = raw_line.strip() + if not value or value.startswith("#"): + continue + if not _valid_seed_url(value): + raise ValueError(f"{path}:{line_number}: invalid HTTP(S) URL: {value!r}") + seeds.append(value) + return seeds + + +def _validate_run_id(value: str) -> str: + """Keep every run directory beneath data/runs and portable across platforms.""" + candidate = (value or "").strip() + if not RUN_ID_PATTERN.fullmatch(candidate) or candidate in {".", ".."}: + raise ValueError( + "run ID must be 1-128 characters using letters, numbers, '.', '_' or '-', " + "and must start with a letter or number" + ) + return candidate + + +def _resolve_seed_urls( + seed_values: list[str] | None, seed_file: str | None, base_dir: str +) -> list[str]: + """Resolve explicit inputs first, then the documented project seed file.""" + seeds: list[str] = [] + if seed_file: + path = os.path.abspath(os.path.expanduser(seed_file)) + if not os.path.isfile(path): + raise ValueError(f"seed file does not exist or is not readable: {path}") + seeds.extend(_load_seed_file(path)) + + for value in seed_values or []: + value = value.strip() + if not _valid_seed_url(value): + if os.path.isfile(os.path.expanduser(value)): + raise ValueError( + f"--seed expects a URL; use --seed-file {value!r} for a file" + ) + raise ValueError(f"invalid HTTP(S) seed URL: {value!r}") + seeds.append(value) + + if not seeds and not seed_file and not seed_values: + # Keep convenient file-only operation, using the path documented in README. + default_file = os.path.join(base_dir, "seed_urls.txt") + legacy_file = os.path.join(base_dir, "OSINTai_FILES1", "seeds", "seed_urls.txt") + for path in (default_file, legacy_file): + if os.path.isfile(path): + seeds.extend(_load_seed_file(path)) + break + + # Ordered de-duplication prevents redundant work without changing priority. + return list(dict.fromkeys(seeds)) + def main(): - ap = argparse.ArgumentParser(description="OSINTai v3.4 FULL (async + proxy + dedupe + embeddings + hunt + graph export)") - ap.add_argument("--seed", help="Seed URL (or use seed_urls.txt file)") + ap = argparse.ArgumentParser( + description=f"OSINTai v{__version__.split('.')[0]} (async crawling and analysis)" + ) + ap.add_argument("--version", action="version", version=f"OSINTai {__version__}") + ap.add_argument( + "--seed", action="append", help="Seed HTTP(S) URL; may be supplied multiple times" + ) + ap.add_argument("--seed-file", help="Text file containing one HTTP(S) seed URL per line") ap.add_argument("--depth", type=int, default=2, help="Max depth") ap.add_argument("--max", type=int, default=150, help="Max URLs") ap.add_argument("--same-domain", action="store_true", help="Only crawl same domain as seed") ap.add_argument("--concurrency", type=int, default=18, help="Global concurrency") ap.add_argument("--per-host", type=int, default=4, help="Per-host concurrency") + ap.add_argument( + "--max-response-bytes", + type=int, + default=2_000_000, + help="Maximum decoded HTML response size (default: 2000000)", + ) ap.add_argument("--ua", default="user_agents.txt", help="User agents file") ap.add_argument("--proxies", default="", help="Optional proxy list file") @@ -47,24 +154,95 @@ def main(): ap.add_argument("--hunt-max", type=int, default=50, help="Max lead URLs per page from hunt mode") ap.add_argument("--run-id", default="", help="Optional run id override") + + ap.add_argument( + "--profile", + default="default", + choices=sorted(RUN_PROFILES), + help="Named run profile applied before explicit flags (default: default)", + ) + ap.add_argument( + "--prompt-profile", + default=STANDARD, + choices=sorted(PAGE_PROFILES), + help="Page analysis lens (default: standard)", + ) + + ap.add_argument( + "--no-analysis", + action="store_true", + help="Skip the post-crawl analysis stage (crawl and report only)", + ) + ap.add_argument( + "--deep", + action="store_true", + help="Optional: run a model-assisted run-level analysis pass", + ) + ap.add_argument( + "--cross-check", + default="", + help="Optional: comma-separated additional Ollama models to cross-check claims against", + ) + ap.add_argument( + "--evaluate", + action="store_true", + help="Optional: score analysis quality against the investigative rubric (deterministic)", + ) + ap.add_argument( + "--training-export", + action="store_true", + help="Optional: export local training/evaluation datasets (requires --evaluate)", + ) + ap.add_argument( + "--gap-days", + type=int, + default=90, + help="Temporal gap threshold in days (default: 90)", + ) + ap.add_argument( + "--leads-per-kind", + type=int, + default=10, + help="Max entities of each kind expanded into pivot leads (default: 10)", + ) + ap.add_argument( + "--experimental-recursive", + type=int, + default=0, + metavar="N", + help="EXPERIMENTAL: extra bounded deep-analysis refinement passes, 0-3 (default: 0)", + ) + + argv = list(sys.argv[1:]) args = ap.parse_args() + # A profile supplies values the operator did not state. An explicit flag always wins. + profile_values = RUN_PROFILES.get(args.profile, {}) + profile_applied = [] + for dest, value in profile_values.items(): + flag = "--" + dest.replace("_", "-") + if flag in argv: + continue + setattr(args, dest, value) + profile_applied.append(f"{dest}={value}") + + if not 0 <= args.experimental_recursive <= 3: + ap.error("--experimental-recursive must be between 0 and 3") + if args.gap_days < 1: + ap.error("--gap-days must be greater than zero") + if args.training_export and not args.evaluate: + ap.error("--training-export requires --evaluate") + base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) - - # Handle seed URLs - check file first, then command line - seed_urls = [] - seed_file = os.path.join(base_dir, "OSINTai_FILES1", "seeds", "seed_urls.txt") - - if os.path.exists(seed_file): - file_seeds = load_lines(seed_file) - if file_seeds: - seed_urls = [url.strip() for url in file_seeds if url.strip() and url.strip().startswith(('http://', 'https://'))] - - # If no seeds from file, use command line argument + + try: + seed_urls = _resolve_seed_urls(args.seed, args.seed_file, base_dir) + except (OSError, ValueError) as exc: + ap.error(str(exc)) if not seed_urls: - if not args.seed: - ap.error("No seed URLs found. Provide --seed URL or add URLs to OSINTai_FILES1/seeds/seed_urls.txt") - seed_urls = [args.seed] + ap.error("No seed URLs found. Provide --seed URL or --seed-file PATH") + if args.max_response_bytes < 1: + ap.error("--max-response-bytes must be greater than zero") print(f"Found {len(seed_urls)} seed URL(s):") for url in seed_urls[:5]: # Show first 5 @@ -73,7 +251,10 @@ def main(): print(f" ... and {len(seed_urls) - 5} more") print() - run_id = args.run_id.strip() or now_run_id() + try: + run_id = _validate_run_id(args.run_id) if args.run_id.strip() else now_run_id() + except ValueError as exc: + ap.error(str(exc)) run_dir = os.path.join(base_dir, "data", "runs", run_id) safe_mkdir(run_dir) @@ -87,7 +268,8 @@ def main(): timeout_s=20.0, min_delay_s=0.2, max_delay_s=1.2, - retries=2 + retries=2, + max_response_bytes=args.max_response_bytes, ) extractor = Extractor() @@ -108,12 +290,29 @@ def main(): model_analyze=args.model, model_embed=args.embed_model, hunt_terms=hunt_terms, - hunt_max_leads=args.hunt_max + hunt_max_leads=args.hunt_max, + prompt_profile=args.prompt_profile ) + cross_check_models = [m.strip() for m in args.cross_check.split(",") if m.strip()] + + # Model reachability is checked once, up front, so an unreachable server is visible + # before the crawl instead of appearing as pages that quietly produced no analysis. + ollama = None + model_status = "OFF" + if not args.no_ollama: + ollama = OllamaAPI() + health = ollama.health() + if not health["reachable"]: + model_status = "UNREACHABLE (analysis will degrade to deterministic only)" + elif not ollama.has_model(args.model): + model_status = f"REACHABLE, model {args.model} NOT FOUND" + else: + model_status = "REACHABLE" + print("") print("=" * 80) - print("OSINTai v3.4 FULL") + print(f"OSINTai v{__version__.split('.')[0]}") print(f"RUN DIR: {run_dir}") print(f"SEEDS: {len(seed_urls)} URL(s)") if len(seed_urls) == 1: @@ -123,18 +322,105 @@ def main(): print(f"ADDITIONAL SEEDS: {len(seed_urls) - 1}") print(f"DEPTH: {args.depth} MAX_URLS: {args.max} SAME_DOMAIN: {args.same_domain}") print(f"CONCURRENCY: {args.concurrency} PER_HOST: {args.per_host}") + print(f"MAX RESPONSE: {args.max_response_bytes} bytes (HTML/XHTML only)") print(f"OLLAMA: {'OFF' if args.no_ollama else args.model}") print(f"EMBED: {'OFF' if args.no_ollama else args.embed_model}") print(f"HUNT: {', '.join(hunt_terms) if hunt_terms else 'OFF'}") + print(f"OLLAMA STATUS: {model_status}") + if args.profile != "default": + print(f"PROFILE: {args.profile}" + (f" ({', '.join(profile_applied)})" if profile_applied else "")) + if args.prompt_profile != STANDARD: + print(f"PROMPT PROFILE: {args.prompt_profile}") + print(f"ANALYSIS: {'OFF' if args.no_analysis else 'ON'}" + f" DEEP: {'ON' if args.deep else 'OFF'}" + f" CROSS-CHECK: {', '.join(cross_check_models) if cross_check_models else 'OFF'}" + f" EVALUATE: {'ON' if args.evaluate else 'OFF'}") print("=" * 80) print("") + started_at = time.time() out = asyncio.run(crawler.crawl()) page_scores = _read_jsonl(out["page_scores_jsonl"]) ranked = rank_pages(page_scores) report_path = write_report(run_dir, ranked) + analysis_report_path = "" + analysis_output = None + if not args.no_analysis: + print("") + print("-" * 80) + print("ANALYSIS") + print("-" * 80) + options = AnalysisOptions( + deep=args.deep, + cross_check_models=cross_check_models, + evaluate=args.evaluate, + training_export=args.training_export, + gap_days=args.gap_days, + prompt_profile=args.prompt_profile, + model=args.model, + use_ollama=(not args.no_ollama), + experimental_recursive=args.experimental_recursive, + max_leads_per_kind=args.leads_per_kind, + ) + try: + analysis_output = analyze_run( + run_dir=run_dir, + options=options, + ollama=ollama, + run_id=run_id, + log=print, + ) + analysis_report_path = write_analysis_report( + run_dir, + analysis_output, + run_id=run_id, + scope={ + "seeds": len(seed_urls), + "depth": args.depth, + "max urls": args.max, + "same domain only": args.same_domain, + "analysis model": args.model if not args.no_ollama else "none", + "prompt profile": args.prompt_profile, + }, + ) + except Exception as exc: + # The crawl is already saved. An analysis failure must not cost the operator it. + print(f"[FAIL] analysis stage aborted -> {exc}") + print(" Crawl artifacts are intact and the ranked report was written.") + + # Run manifest: what was asked for, what answered, and where it went. + write_json(os.path.join(run_dir, "run_manifest.json"), { + "run_id": run_id, + "osintai_version": __version__, + "started_at": started_at, + "finished_at": time.time(), + "seed_urls": seed_urls, + "profile": args.profile, + "prompt_profile": args.prompt_profile, + "depth": args.depth, + "max_urls": args.max, + "same_domain_only": args.same_domain, + "concurrency": args.concurrency, + "per_host": args.per_host, + "max_response_bytes": args.max_response_bytes, + "hunt_terms": hunt_terms, + "ollama_enabled": not args.no_ollama, + "ollama_status": model_status, + "analysis_model": args.model, + "embed_model": args.embed_model, + "analysis_enabled": not args.no_analysis, + "deep": args.deep, + "cross_check_models": cross_check_models, + "evaluate": args.evaluate, + "training_export": args.training_export, + "experimental_recursive": args.experimental_recursive, + "gap_days": args.gap_days, + "pages_scored": len(page_scores), + "analysis_stats": analysis_output.stats if analysis_output else {}, + }) + print("") print("DONE.") print(f"- URLs: {out['urls_jsonl']}") @@ -145,6 +431,28 @@ def main(): print(f"- Graph nodes: {out['graph_nodes']}") print(f"- Graph edges: {out['graph_edges']}") print(f"- Report: {report_path}") + if analysis_output is not None: + artifacts = analysis_output.artifacts + for label, key in ( + ("Findings", "findings"), + ("Correlations", "correlations"), + ("Timeline", "timeline"), + ("Hypotheses", "hypotheses"), + ("Leads", "leads"), + ("Analysis:", "analysis_summary"), + ): + if artifacts.get(key): + print(f"- {label + ':' if not label.endswith(':') else label:<13} {artifacts[key]}") + if artifacts.get("training_export"): + print(f"- Training export: {artifacts['training_export']}") + if analysis_report_path: + print(f"- Analysis report: {analysis_report_path}") + stats = analysis_output.stats + print("") + print(f" Entities: {stats.get('entities', 0)}" + f" Findings: {stats.get('finding_count', 0)}" + f" Hypotheses: {stats.get('hypothesis_count', 0)}" + f" Leads: {stats.get('lead_count', 0)}") print("") if __name__ == "__main__": diff --git a/src/osintai/correlation.py b/src/osintai/correlation.py new file mode 100644 index 0000000..976c488 --- /dev/null +++ b/src/osintai/correlation.py @@ -0,0 +1,339 @@ +"""Cross-source correlation for OSINTai. + +Two identifiers are related when the crawl produced evidence that they are related, and the +evidence travels with the link. Everything here is a scored *candidate*; nothing merges two +entities into one, because an entity merge made on a co-occurrence is not reversible once it +has propagated through a report. + +Relationships are built from shared source evidence, never from coincidental list positions. +That preserves reversibility and prevents sorted data from manufacturing identities. +""" + +from __future__ import annotations + +from collections import Counter, defaultdict +from dataclasses import dataclass, field +from typing import Any, Dict, Iterable, List, Optional, Tuple +from urllib.parse import urlparse + +from .entities import ( + DOMAIN, + EMAIL, + Entity, + EntityIndex, + NAME, + PHONE, + USERNAME, + canonical, +) +from .provenance import ( + DERIVED, + HIGH, + LOW, + MEDIUM, + CheckResult, + Finding, + deterministic_confidence, + source_support, +) + +# Relationship types +CO_OCCURS = "co_occurs_with" +SHARES_DOMAIN = "shares_domain_with" +LOCAL_PART_MATCH = "local_part_matches_handle" +NAME_MATCH = "name_matches_handle" +SAME_NUMBER = "same_number_written_differently" + + +@dataclass +class Correlation: + """A candidate relationship between two entities, with the evidence that produced it.""" + + left_kind: str + left: str + right_kind: str + right: str + relation: str + evidence_urls: List[str] = field(default_factory=list) + rationale: str = "" + score: float = 0.0 + status: str = "CANDIDATE" + + def to_dict(self) -> Dict[str, Any]: + return { + "status": self.status, + "relation": self.relation, + "left": {"kind": self.left_kind, "value": self.left}, + "right": {"kind": self.right_kind, "value": self.right}, + "evidence_urls": self.evidence_urls, + "evidence_count": len(self.evidence_urls), + "rationale": self.rationale, + "score": round(self.score, 4), + } + + +def _registrable(host: str) -> str: + """Last two labels of a hostname. Good enough to group a site's own subdomains.""" + parts = [p for p in (host or "").split(".") if p] + return ".".join(parts[-2:]) if len(parts) >= 2 else (host or "") + + +def map_domains_to_urls(indicator_rows: Iterable[Dict[str, Any]]) -> Dict[str, Any]: + """Group crawled URLs under the domains that reference them, ranked by frequency.""" + domain_map: Dict[str, List[str]] = defaultdict(list) + frequency: Counter = Counter() + + for row in indicator_rows or []: + source = row.get("url") or "" + for url in row.get("urls") or []: + try: + host = (urlparse(url).hostname or "").lower() + except ValueError: + continue + if not host: + continue + if url not in domain_map[host]: + domain_map[host].append(url) + frequency[host] += 1 + source_host = (row.get("domain") or "").lower() + if source_host and source not in domain_map[source_host]: + domain_map[source_host].append(source) + + return { + "total_domains": len(domain_map), + "top_domains": frequency.most_common(25), + "domain_map": {d: urls[:200] for d, urls in domain_map.items()}, + } + + +# An identifier present on this fraction of the crawled pages is site furniture — a +# footer contact, a masthead handle — not a per-page signal. +UBIQUITY_FRACTION = 0.25 + +# Pages carrying an enormous identifier set (directories, dumps, member lists) are skipped +# for pairing: everything co-occurs with everything there and the links mean nothing. +MAX_IDENTIFIERS_PER_PAGE = 60 + + +def _site_wide(index: EntityIndex, page_count: int) -> set: + """Entity keys that appear across so much of the crawl they carry no pairing signal.""" + if page_count < 8: + return set() + threshold = max(3, int(page_count * UBIQUITY_FRACTION)) + return {e.key for e in index.all() if len(e.sources) >= threshold} + + +def _co_occurrence_pairs( + index: EntityIndex, page_count: int +) -> Tuple[List[Correlation], int]: + """Identifiers that appeared together on the same page. + + Site-wide identifiers are excluded from pairing. On a single-site crawl the contact + address in the footer co-occurs with every handle on the site, and pairing them produces + hundreds of links that describe the page template rather than the subject. + """ + by_source = index.by_source() + pair_evidence: Dict[Tuple[Tuple[str, str], Tuple[str, str]], List[str]] = defaultdict(list) + linkable = {EMAIL, PHONE, USERNAME, NAME} + ubiquitous = _site_wide(index, page_count) + + for url, entities in by_source.items(): + interesting = [ + e for e in entities if e.kind in linkable and e.key not in ubiquitous + ] + if len(interesting) < 2 or len(interesting) > MAX_IDENTIFIERS_PER_PAGE: + continue + ordered = sorted(interesting, key=lambda e: (e.kind, e.canonical_value)) + for i, left in enumerate(ordered): + for right in ordered[i + 1:]: + key = (left.key, right.key) + if url not in pair_evidence[key]: + pair_evidence[key].append(url) + + correlations: List[Correlation] = [] + for (left_key, right_key), urls in pair_evidence.items(): + # A single shared page is weak; two or more independent pages is a real signal. + score = min(0.85, 0.25 + 0.2 * (len(urls) - 1)) + correlations.append(Correlation( + left_kind=left_key[0], left=left_key[1], + right_kind=right_key[0], right=right_key[1], + relation=CO_OCCURS, + evidence_urls=urls[:25], + rationale=( + f"Both identifiers appear on {len(urls)} crawled page(s). Co-occurrence on a " + "page is not proof of a shared owner." + ), + score=score, + )) + return correlations, len(ubiquitous) + + +def _identity_hints(index: EntityIndex) -> List[Correlation]: + """Shape-based identity candidates: email local parts, name forms, phone variants.""" + correlations: List[Correlation] = [] + handles = {e.canonical_value.lstrip("@"): e for e in index.of_kind(USERNAME)} + + for email in index.of_kind(EMAIL): + local = email.canonical_value.split("@", 1)[0] + if len(local) < 4: + continue + handle = handles.get(local) + if handle is None: + continue + shared = sorted(set(email.sources) & set(handle.sources)) + correlations.append(Correlation( + left_kind=EMAIL, left=email.canonical_value, + right_kind=USERNAME, right=handle.value, + relation=LOCAL_PART_MATCH, + evidence_urls=(shared or sorted(set(email.sources) | set(handle.sources)))[:25], + rationale=( + f"Email local part {local!r} matches the handle. Common local parts are reused " + "widely by unrelated people; treat as a pivot, not an identity." + ), + score=0.55 if shared else 0.35, + )) + + for name in index.of_kind(NAME): + collapsed = name.canonical_value.replace(" ", "") + handle = handles.get(collapsed) + if handle is None: + continue + shared = sorted(set(name.sources) & set(handle.sources)) + correlations.append(Correlation( + left_kind=NAME, left=name.value, + right_kind=USERNAME, right=handle.value, + relation=NAME_MATCH, + evidence_urls=(shared or sorted(set(name.sources) | set(handle.sources)))[:25], + rationale="Handle matches the name with separators removed.", + score=0.4 if shared else 0.25, + )) + + # Phone entities fold to a canonical digit key, so differently written forms of one + # number land on one entity. This reports only where that genuinely happened: the claim + # rests on the forms actually seen, not on the forms the number could be written in. + for phone in index.of_kind(PHONE): + if len(phone.observed_forms) > 1: + correlations.append(Correlation( + left_kind=PHONE, left=phone.canonical_value, + right_kind=PHONE, right=", ".join(phone.observed_forms[:4]), + relation=SAME_NUMBER, + evidence_urls=list(phone.sources)[:25], + rationale=( + f"One number written {len(phone.observed_forms)} different ways across the " + "crawl; these are the same number." + ), + score=0.8, + )) + + return correlations + + +def _domain_families(index: EntityIndex) -> List[Correlation]: + """Hosts sharing a registrable domain.""" + families: Dict[str, List[Entity]] = defaultdict(list) + for entity in index.of_kind(DOMAIN): + base = _registrable(entity.canonical_value) + if base and base != entity.canonical_value: + families[base].append(entity) + + correlations: List[Correlation] = [] + for base, members in families.items(): + if len(members) < 2: + continue + sources: List[str] = [] + for member in members: + for url in member.sources: + if url not in sources: + sources.append(url) + correlations.append(Correlation( + left_kind=DOMAIN, left=base, + right_kind=DOMAIN, right=", ".join(sorted(m.canonical_value for m in members)[:8]), + relation=SHARES_DOMAIN, + evidence_urls=sources[:25], + rationale=f"{len(members)} hosts share the registrable domain {base}.", + score=0.9, + )) + return correlations + + +# The full candidate set goes to the artifact; the report gets the top slice. A findings +# list nobody can read through is a findings list nobody reads. +MAX_CORRELATION_FINDINGS = 60 +MAX_CORRELATION_ROWS = 20000 + + +def correlate( + index: EntityIndex, indicator_rows: Iterable[Dict[str, Any]], page_count: int = 0 +) -> CheckResult: + """Run every correlation method and report the candidates.""" + result = CheckResult(check_name="Cross-Source Correlation") + + rows = list(indicator_rows or []) + page_count = page_count or len(rows) + domain_mapping = map_domains_to_urls(rows) + + correlations: List[Correlation] = [] + correlations.extend(_domain_families(index)) + correlations.extend(_identity_hints(index)) + co_occurrences, site_wide_count = _co_occurrence_pairs(index, page_count) + correlations.extend(co_occurrences) + + correlations.sort(key=lambda c: (-c.score, c.relation, c.left)) + truncated = max(0, len(correlations) - MAX_CORRELATION_ROWS) + result.rows = [c.to_dict() for c in correlations[:MAX_CORRELATION_ROWS]] + result.stats = { + "correlation_count": len(correlations), + "rows_written": len(result.rows), + "rows_truncated": truncated, + "site_wide_identifiers_excluded": site_wide_count, + "total_domains": domain_mapping["total_domains"], + "top_domains": domain_mapping["top_domains"][:10], + } + + # Only the strongest candidates become findings; the rest stay available in the artifact. + promoted = 0 + for correlation in correlations: + if correlation.score < 0.5: + continue + if promoted >= MAX_CORRELATION_FINDINGS: + break + promoted += 1 + priority = HIGH if correlation.score >= 0.8 else MEDIUM + result.findings.append(Finding( + check="Correlation Candidate", + item=f"{correlation.left} <-> {correlation.right}", + reason=f"{correlation.relation}: {correlation.rationale}", + next_step=( + "Open the evidence pages and establish whether these identifiers share an owner. " + "Record the answer; do not treat the candidate link as established." + ), + origin=DERIVED, + priority=priority, + sources=correlation.evidence_urls, + evidence=correlation.to_dict(), + method=correlation.relation, + confidence=[ + deterministic_confidence(correlation.score, correlation.rationale), + source_support(correlation.evidence_urls), + ], + )) + + eligible = sum(1 for c in correlations if c.score >= 0.5) + result.notes.append( + f"Built {len(correlations)} candidate correlation(s) across {len(index)} entities " + f"and {domain_mapping['total_domains']} domain(s). All links are candidates; no entities " + "were merged." + ) + if site_wide_count: + result.notes.append( + f"Excluded {site_wide_count} site-wide identifier(s) from co-occurrence pairing; " + "they appear across too much of the crawl to indicate a specific relationship." + ) + if eligible > promoted: + result.notes.append( + f"Promoted the {promoted} strongest candidate(s) to findings; the remaining " + f"{eligible - promoted} scoring candidate(s) are in correlations.jsonl." + ) + if truncated: + result.notes.append(f"{truncated} lowest-scoring candidate(s) omitted from the artifact.") + return result diff --git a/src/osintai/crawler.py b/src/osintai/crawler.py index a40a8c7..b57d56e 100644 --- a/src/osintai/crawler.py +++ b/src/osintai/crawler.py @@ -9,15 +9,16 @@ import httpx -from .normalize import clean_url, same_domain +from .normalize import clean_url from .storage import sha1, safe_mkdir, append_jsonl, read_json, write_json from .extractor import Extractor -from .fetcher import AsyncFetcher +from .fetcher import AsyncFetcher, FetchRejected from .ollama_api import OllamaAPI from .dedupe import sha1_text, simhash_64, hamming64 from .analyzer import compute_page_signal from .hunt import hunt_leads from .graph_export import export_graph +from .prompts import STANDARD, page_prompt def is_probably_html(resp: httpx.Response) -> bool: ctype = resp.headers.get("content-type", "") or "" @@ -25,8 +26,8 @@ def is_probably_html(resp: httpx.Response) -> bool: def host_of(url: str) -> str: try: - return urlparse(url).netloc.lower() - except: + return (urlparse(url).hostname or "").lower() + except ValueError: return "" @dataclass @@ -59,10 +60,12 @@ def __init__( model_analyze: str, model_embed: str, hunt_terms: List[str], - hunt_max_leads: int + hunt_max_leads: int, + prompt_profile: str = STANDARD ): + self.prompt_profile = prompt_profile self.seed_urls = seed_urls - self.primary_seed = seed_urls[0] if seed_urls else "" # For same_domain comparison + self.allowed_seed_hosts = {host_of(url) for url in seed_urls if host_of(url)} self.max_depth = max_depth self.max_urls = max_urls self.run_dir = run_dir @@ -150,8 +153,8 @@ def _save_state(self): if getattr(self.fetcher, "proxy_pool", None): try: write_json(self.proxy_state, self.fetcher.proxy_pool.as_dict()) - except: - pass + except (OSError, TypeError, ValueError) as exc: + print(f"[WARN] could not save proxy state: {exc}") def _enqueue(self, url: str, depth: int) -> bool: url = clean_url(url) @@ -174,7 +177,16 @@ def _host_lock(self, url: str) -> asyncio.Semaphore: def _scoped(self, url: str) -> bool: if not url: return False - if self.same_domain_only and not same_domain(url, self.primary_seed): + try: + parsed = urlparse(url) + host = (parsed.hostname or "").lower() + except ValueError: + return False + if parsed.scheme not in {"http", "https"} or parsed.username or parsed.password: + return False + if not host: + return False + if self.same_domain_only and host not in self.allowed_seed_hosts: return False return True @@ -188,34 +200,9 @@ def _dedupe_near(self, text: str) -> Tuple[bool, int]: return False, sh def _analysis_prompt(self, url: str, title: str, text: str) -> str: - snippet = (text or "")[:12000] - return f""" -You are an OSINT analyst. Produce a structured intelligence extraction from this webpage. - -Rules: -- No filler. No moralizing. -- Only use evidence from the content. -- Output valid JSON only. - -Return schema: -{{ - "url": "...", - "title": "...", - "summary": "2-4 sentences", - "key_entities": ["..."], - "key_locations": ["..."], - "key_dates": ["..."], - "keywords": ["..."], - "risk_flags": ["..."], - "actionable_leads": ["..."] -}} - -URL: {url} -TITLE: {title} - -CONTENT: -{snippet} -""".strip() + # Delegates to the prompt registry. The 'standard' profile is the same text this + # method has always produced, so osint-tuned-v3 sees identical input. + return page_prompt(url, title, text, self.prompt_profile) async def _process_one(self, client: httpx.AsyncClient, url: str, depth: int): url = clean_url(url) @@ -231,11 +218,24 @@ async def _process_one(self, client: httpx.AsyncClient, url: str, depth: int): lock = self._host_lock(url) async with lock: try: - resp = await self.fetcher.get(client, url) + resp = await self.fetcher.get(client, url, redirect_validator=self._scoped) + except FetchRejected as e: + self.visited.add(url) + print(f"[SKIP] {url} -> {e}") + return except Exception as e: print(f"[FAIL] {url} -> {e}") return + requested_url = url + final_url = clean_url(str(resp.url)) + if not final_url or not self._scoped(final_url): + self.visited.add(url) + print(f"[SKIP] redirect outside seed domains: {url} -> {final_url}") + return + if final_url != requested_url: + self.visited.add(requested_url) + url = final_url status = resp.status_code if status != 200 or not is_probably_html(resp): self.visited.add(url) @@ -305,8 +305,8 @@ async def _process_one(self, client: httpx.AsyncClient, url: str, depth: int): out_vec = os.path.join(self.embed_dir, f"{rid}.embed.json") with open(out_vec, "w", encoding="utf-8") as f: json.dump({"url": url, "model": self.model_embed, "embedding": vec}, f) - except: - pass + except (OSError, TypeError, ValueError) as exc: + print(f"[WARN] embedding write failed for {url}: {exc}") # hunt mode on every page (lightweight) hunt = hunt_leads(text, self.hunt_terms, max_leads=self.hunt_max_leads) if self.hunt_terms else {"hits": [], "lead_urls": []} @@ -331,12 +331,20 @@ async def _process_one(self, client: httpx.AsyncClient, url: str, depth: int): async def crawl(self): limits = httpx.Limits(max_connections=self.concurrency * 2, max_keepalive_connections=self.concurrency) - async with httpx.AsyncClient(limits=limits) as client: + async with httpx.AsyncClient(limits=limits, trust_env=False) as client: sem = asyncio.Semaphore(self.concurrency) async def worker(u: str, d: int): async with sem: - await self._process_one(client, u, d) + try: + await self._process_one(client, u, d) + except asyncio.CancelledError: + raise + except Exception as exc: + # Page-specific parser/analyzer defects must not abort + # unrelated work in the same concurrent batch. + self.visited.add(u) + print(f"[FAIL] {u} -> unexpected processing error: {exc}") while self.queue and len(self.visited) < self.max_urls: batch = [] diff --git a/src/osintai/dedupe.py b/src/osintai/dedupe.py index 847a653..f3091d0 100644 --- a/src/osintai/dedupe.py +++ b/src/osintai/dedupe.py @@ -2,10 +2,14 @@ import re def sha1_bytes(b: bytes) -> str: - return hashlib.sha1(b).hexdigest() + # Stable legacy artifact identifier; never used for cryptographic security. + return hashlib.sha1(b, usedforsecurity=False).hexdigest() def sha1_text(s: str) -> str: - return hashlib.sha1(s.encode("utf-8", errors="ignore")).hexdigest() + # Stable legacy artifact identifier; never used for cryptographic security. + return hashlib.sha1( + s.encode("utf-8", errors="ignore"), usedforsecurity=False + ).hexdigest() def simhash_64(text: str) -> int: """ diff --git a/src/osintai/entities.py b/src/osintai/entities.py new file mode 100644 index 0000000..16bc147 --- /dev/null +++ b/src/osintai/entities.py @@ -0,0 +1,321 @@ +"""Entity typing, normalization and cross-page identity for OSINTai. + +The crawler's Extractor mines a page for indicator strings. This module turns those strings +into typed, normalized entities that can be compared across pages, which is what correlation +and lead generation need. + +Identifiers are typed from their shape and normalized before comparison. Phone variants are +kept separately from observed forms so the same number written four ways is one entity +without claiming that an unobserved representation appeared in source material. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple + +# Identifier types +EMAIL = "email" +PHONE = "phone" +NAME = "name" +USERNAME = "username" +DOMAIN = "domain" +IP = "ip" +URL = "url" +CRYPTO_BTC = "btc" +CRYPTO_ETH = "eth" +DATE = "date" +ADDRESS = "address" +SECRET_KIND = "secret" + +IDENTIFIER_TYPES = ( + EMAIL, PHONE, NAME, USERNAME, DOMAIN, IP, URL, + CRYPTO_BTC, CRYPTO_ETH, DATE, ADDRESS, SECRET_KIND, +) + +_EMAIL_SHAPE = re.compile(r"^[^\s@]+@[^\s@]+\.[^\s@]+$") +_PHONE_SHAPE = re.compile(r"^[+\d\s().-]+$") +_NAME_SHAPE = re.compile(r"^[a-z][a-z .'-]+\s+[a-z .'-]+$", re.IGNORECASE) + +# Extended indicator classes used after the crawl. Broad path matching is deliberately +# excluded because it would match almost any slash-separated text. +DATE_RE = re.compile(r"\b(?:\d{4}-\d{2}-\d{2}|\d{2}/\d{2}/\d{4})\b") +NAME_CANDIDATE_RE = re.compile(r"\b[A-Z][a-z]{1,20} [A-Z][a-z]{1,20}\b") +ADDRESS_RE = re.compile( + r"\b\d{1,5} [A-Za-z0-9.\s]{2,40}?" + r"(?:Street|St|Avenue|Ave|Boulevard|Blvd|Road|Rd|Lane|Ln|Drive|Dr|Court|Ct|Way|Terrace|Ter)\b" +) +API_TOKEN_RE = re.compile( + r"(?i)(?:api[_-]?key|apikey|access[_-]?token|auth[_-]?token|secret|token)" + r"[\"'\s:=]{1,5}[\"']?([A-Za-z0-9_\-]{16,64})[\"']?" +) +JWT_RE = re.compile(r"\b(eyJ[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{5,})\b") + +# The crawler's own domain and handle patterns are ASCII-only, which means a lookalike +# domain built from Cyrillic or fullwidth characters is invisible to it — precisely the +# case homoglyph analysis exists to catch. These patterns accept non-ASCII letters so the +# check has something to inspect. Kept here rather than in Extractor so the crawl hot path +# and its output schema stay exactly as they were. +UNICODE_DOMAIN_RE = re.compile( + r"(?:[^\W\d_]|[a-zA-Z0-9-])+(?:\.(?:[^\W\d_]|[a-zA-Z0-9-])+)+", re.UNICODE +) +UNICODE_HANDLE_RE = re.compile(r"(? bool: + return any(ord(ch) > 0x7E for ch in value) +CREDENTIAL_PAIR_RE = re.compile( + r"(?m)^[ \t]*([\w.+-]+@[\w.-]+\.[A-Za-z]{2,}|[\w.-]{3,32}):(?!//)(\S{6,64})[ \t]*$" +) + +# Left-hand values that make a `word:value` line something other than a credential. Without +# these a bare URL on its own line reads as "user https" with a six-character secret. +_NOT_CREDENTIAL_KEYS = { + "http", "https", "ftp", "ftps", "mailto", "tel", "data", "file", "ws", "wss", + "note", "notes", "source", "sources", "url", "link", "links", "ref", "see", + "date", "time", "subject", "from", "to", "cc", "bcc", "re", "via", "id", + "version", "type", "name", "title", "author", "tags", "category", "status", +} + + +def detect_type(value: str) -> str: + """Best-effort identifier typing from shape alone. + + Order matters: an email is unambiguous, a mostly-digit string is a phone, two + capitalized words are a name, and everything else falls through to username. + """ + v = (value or "").strip() + if not v: + return USERNAME + if _EMAIL_SHAPE.match(v): + return EMAIL + digit_count = sum(1 for ch in v if ch.isdigit()) + if digit_count >= 7 and _PHONE_SHAPE.match(v): + return PHONE + if "@" not in v and _NAME_SHAPE.match(v): + return NAME + return USERNAME + + +def normalize_phone(raw: str) -> Dict[str, str]: + """Reduce a phone string to its digits and E.164-ish form.""" + trimmed = (raw or "").strip() + has_plus = trimmed.startswith("+") + digits = "".join(ch for ch in trimmed if ch.isdigit()) + return { + "digits": digits, + "e164": f"+{digits}" if has_plus and digits else digits, + "has_plus": "yes" if has_plus else "no", + } + + +def phone_variants(raw: str) -> List[str]: + """Every common written form of a US-style number, so page-to-page matching works.""" + digits = normalize_phone(raw)["digits"] + variants: Set[str] = set() + if (raw or "").strip(): + variants.add(raw.strip()) + if digits: + variants.add(digits) + ten = digits[1:] if len(digits) == 11 and digits.startswith("1") else digits + if len(ten) == 10: + a, b, c = ten[:3], ten[3:6], ten[6:] + variants.update({ + f"({a}) {b}-{c}", + f"{a}-{b}-{c}", + f"{a}.{b}.{c}", + f"{a} {b} {c}", + f"+1{ten}", + ten, + }) + return sorted(v for v in variants if v) + + +def canonical(value: str, kind: str) -> str: + """The comparison key for an entity of a given kind. + + Two entities are the same entity when their canonical forms match. Everything that is + case- or punctuation-insensitive gets folded here so correlation does not have to know + the rules for each type. + """ + v = (value or "").strip() + if not v: + return "" + if kind in (EMAIL, DOMAIN, USERNAME, CRYPTO_ETH, URL): + return v.lower().strip(".") + if kind == PHONE: + digits = normalize_phone(v)["digits"] + return digits[1:] if len(digits) == 11 and digits.startswith("1") else digits + if kind == NAME: + return " ".join(v.split()).lower() + return v + + +@dataclass +class Entity: + """A typed identifier and every page it was observed on. + + Observations accumulate; they are never merged away. The source list is what later lets + a correlation say why it thinks two things are related. + """ + + kind: str + value: str + canonical_value: str = "" + sources: List[str] = field(default_factory=list) + variants: List[str] = field(default_factory=list) + observed_forms: List[str] = field(default_factory=list) + count: int = 0 + + def __post_init__(self) -> None: + if not self.canonical_value: + self.canonical_value = canonical(self.value, self.kind) + + @property + def key(self) -> Tuple[str, str]: + return (self.kind, self.canonical_value) + + def observe(self, source: str, raw: str = "") -> None: + """Record one sighting. + + `observed_forms` holds the literal strings actually seen, which is separate from + `variants` (forms this identifier *could* be written in). Only the observed list can + support a claim about how something was written. + """ + self.count += 1 + if source and source not in self.sources: + self.sources.append(source) + raw = (raw or "").strip() + if raw and raw not in self.observed_forms and len(self.observed_forms) < 12: + self.observed_forms.append(raw) + + def to_dict(self) -> Dict[str, Any]: + return { + "kind": self.kind, + "value": self.value, + "canonical": self.canonical_value, + "sources": self.sources, + "source_count": len(self.sources), + "variants": self.variants, + "observed_forms": self.observed_forms, + "count": self.count, + } + + +class EntityIndex: + """All entities seen in a run, keyed by (kind, canonical value).""" + + def __init__(self) -> None: + self._entities: Dict[Tuple[str, str], Entity] = {} + + def add(self, kind: str, value: str, source: str) -> Optional[Entity]: + key_value = canonical(value, kind) + if not key_value: + return None + key = (kind, key_value) + entity = self._entities.get(key) + if entity is None: + entity = Entity(kind=kind, value=value, canonical_value=key_value) + if kind == PHONE: + entity.variants = phone_variants(value) + self._entities[key] = entity + entity.observe(source, raw=value) + return entity + + def add_many(self, kind: str, values: Iterable[str], source: str) -> None: + for value in values or []: + self.add(kind, value, source) + + def get(self, kind: str, value: str) -> Optional[Entity]: + return self._entities.get((kind, canonical(value, kind))) + + def of_kind(self, kind: str) -> List[Entity]: + return [e for e in self._entities.values() if e.kind == kind] + + def all(self) -> List[Entity]: + return list(self._entities.values()) + + def multi_source(self, minimum: int = 2) -> List[Entity]: + """Entities corroborated across several pages — the ones worth attention first.""" + return sorted( + (e for e in self._entities.values() if len(e.sources) >= minimum), + key=lambda e: (-len(e.sources), e.kind, e.canonical_value), + ) + + def by_source(self) -> Dict[str, List[Entity]]: + mapping: Dict[str, List[Entity]] = {} + for entity in self._entities.values(): + for source in entity.sources: + mapping.setdefault(source, []).append(entity) + return mapping + + def __len__(self) -> int: + return len(self._entities) + + +# Maps the indicator keys the crawler already writes onto entity kinds. Anything the +# extractor gains later only needs a line here to become correlatable. +INDICATOR_KIND_MAP = { + "emails": EMAIL, + "phones": PHONE, + "domains": DOMAIN, + "ip_addresses": IP, + "urls": URL, + "btc_addresses": CRYPTO_BTC, + "eth_addresses": CRYPTO_ETH, + "social_handles": USERNAME, + "dates": DATE, + "name_candidates": NAME, + "addresses": ADDRESS, +} + + +def index_indicators(rows: Iterable[Dict[str, Any]]) -> EntityIndex: + """Build an entity index from the indicator records the crawl already produced.""" + index = EntityIndex() + for row in rows or []: + source = row.get("url") or "" + for indicator_key, kind in INDICATOR_KIND_MAP.items(): + values = row.get(indicator_key) + if isinstance(values, list): + index.add_many(kind, values, source) + return index + + +def extract_extended(text: str) -> Dict[str, List[str]]: + """Indicator classes beyond the crawler's built-in set. + + Kept separate from Extractor so the crawl hot path and its existing output schema are + untouched; the analysis stage calls this over already-saved page text. + """ + body = text or "" + dates = sorted(set(DATE_RE.findall(body)))[:200] + name_candidates = sorted(set(NAME_CANDIDATE_RE.findall(body)))[:200] + addresses = sorted({m.strip() for m in ADDRESS_RE.findall(body)})[:100] + api_tokens = sorted(set(API_TOKEN_RE.findall(body)))[:100] + jwts = sorted(set(JWT_RE.findall(body)))[:50] + credential_pairs = [ + f"{user}:{secret}" + for user, secret in CREDENTIAL_PAIR_RE.findall(body) + if user.lower() not in _NOT_CREDENTIAL_KEYS + ][:100] + # Only the non-ASCII ones are worth carrying: the ASCII domains and handles are already + # in the crawler's own indicator output. + unicode_domains = sorted({ + m.strip(".") for m in UNICODE_DOMAIN_RE.findall(body) if _has_non_ascii(m) + })[:100] + unicode_handles = sorted({ + "@" + m for m in UNICODE_HANDLE_RE.findall(body) if _has_non_ascii(m) + })[:100] + + return { + "dates": dates, + "name_candidates": name_candidates, + "addresses": addresses, + "api_tokens": api_tokens, + "jwts": jwts, + "credential_pairs": credential_pairs, + "unicode_domains": unicode_domains, + "unicode_handles": unicode_handles, + } diff --git a/src/osintai/evaluation.py b/src/osintai/evaluation.py new file mode 100644 index 0000000..c5d4296 --- /dev/null +++ b/src/osintai/evaluation.py @@ -0,0 +1,298 @@ +"""Deterministic evaluation of model-assisted analysis quality. + +The rubric measures entity accuracy, confidence language, source discipline, pivot +reasoning, and format compliance. Every check is computed in code against the page text the +analysis claims to describe, so the score is reproducible and stable across runs. Recurring +failure modes are aggregated because repeated weakness is more actionable than one outlier. +""" + +from __future__ import annotations + +import re +from collections import Counter +from typing import Any, Callable, Dict, List, Optional, Tuple + +from .provenance import ( + DERIVED, + HIGH, + LOW, + MEDIUM, + CheckResult, + Finding, + deterministic_confidence, +) + +# Stable rubric weights; tests enforce that they remain normalized. +RUBRIC_WEIGHTS = { + "entity_accuracy": 0.30, + "confidence_language": 0.25, + "source_discipline": 0.20, + "pivot_reasoning": 0.15, + "format_compliance": 0.10, +} + +REQUIRED_KEYS = ( + "url", "title", "summary", "key_entities", "key_locations", + "key_dates", "keywords", "risk_flags", "actionable_leads", +) + +LIST_KEYS = ( + "key_entities", "key_locations", "key_dates", + "keywords", "risk_flags", "actionable_leads", +) + +# Language that asserts more than a single crawled page can support. +OVERSTATED = ( + "confirmed", "verified", "proven", "definitely", "certainly", "undoubtedly", + "without a doubt", "conclusively", "establishes that", "proves", +) + +# Language that correctly marks a claim as provisional. +HEDGED = ( + "appears", "indicated", "possible", "possibly", "unverified", "suggests", + "may ", "might ", "reportedly", "claims", "requires confirmation", "unclear", + "potential", "likely", +) + +# A lead that is only a verb is not a lead. +VAGUE_LEADS = ( + "investigate further", "further investigation", "more research", "look into it", + "additional analysis", "review the site", "monitor", "keep an eye", +) + + +def _normalize(text: str) -> str: + return re.sub(r"\s+", " ", (text or "").lower()) + + +def score_entity_accuracy(analysis: Dict[str, Any], page_text: str) -> Tuple[float, List[str]]: + """Do the entities the model reported actually occur in the page it analyzed?""" + entities = [e for e in (analysis.get("key_entities") or []) if isinstance(e, str) and e.strip()] + if not entities: + return 1.0, [] # Nothing claimed is not an accuracy failure. + if not page_text: + return 0.5, ["entity_accuracy: page text unavailable for verification"] + + haystack = _normalize(page_text) + missing = [e for e in entities if _normalize(e) not in haystack] + grounded = len(entities) - len(missing) + score = grounded / len(entities) + failures = [] + if missing: + failures.append( + f"entity_accuracy: {len(missing)}/{len(entities)} reported entities not found in page text" + ) + return score, failures + + +def score_confidence_language(analysis: Dict[str, Any]) -> Tuple[float, List[str]]: + """Penalize overstatement, credit appropriate hedging.""" + text = _normalize( + " ".join([ + str(analysis.get("summary") or ""), + " ".join(str(f) for f in (analysis.get("risk_flags") or [])), + ]) + ) + if not text.strip(): + return 1.0, [] + + overstated = [term for term in OVERSTATED if term in text] + hedged = [term for term in HEDGED if term in text] + + score = 1.0 + failures = [] + if overstated: + score -= min(0.7, 0.25 * len(overstated)) + failures.append(f"confidence_language: overstated terms used ({', '.join(overstated[:4])})") + if not hedged and len(text) > 200: + score -= 0.15 + failures.append("confidence_language: no qualifying language in a substantive summary") + return max(0.0, score), failures + + +def score_source_discipline(analysis: Dict[str, Any], page_text: str) -> Tuple[float, List[str]]: + """Is the analysis anchored to the page it was given?""" + failures = [] + score = 1.0 + + if not str(analysis.get("url") or "").strip(): + score -= 0.4 + failures.append("source_discipline: analysis carries no source URL") + + summary = str(analysis.get("summary") or "") + if summary and page_text: + # Content words from the summary that never appear in the page suggest the model + # drew on something other than the source it was given. + words = {w for w in re.findall(r"[a-z]{6,}", summary.lower())} + haystack = _normalize(page_text) + if words: + unsupported = sum(1 for w in words if w not in haystack) + ratio = unsupported / len(words) + if ratio > 0.6: + score -= 0.4 + failures.append( + f"source_discipline: {ratio:.0%} of summary content words absent from page text" + ) + elif not page_text: + failures.append("source_discipline: page text unavailable for verification") + score -= 0.1 + + return max(0.0, score), failures + + +def score_pivot_reasoning(analysis: Dict[str, Any]) -> Tuple[float, List[str]]: + """Are the leads specific enough to act on?""" + leads = analysis.get("actionable_leads") or [] + flat: List[str] = [] + for lead in leads: + if isinstance(lead, str): + flat.append(lead) + elif isinstance(lead, dict): + flat.append(str(lead.get("lead") or lead.get("description") or "")) + flat = [l for l in flat if l.strip()] + + if not flat: + return 0.6, ["pivot_reasoning: no actionable leads proposed"] + + vague = [l for l in flat if any(v in l.lower() for v in VAGUE_LEADS) and len(l) < 90] + specific = len(flat) - len(vague) + score = specific / len(flat) + failures = [] + if vague: + failures.append(f"pivot_reasoning: {len(vague)}/{len(flat)} leads are generic restatements") + return score, failures + + +def score_format_compliance(analysis: Dict[str, Any]) -> Tuple[float, List[str]]: + """Did the model return the schema it was asked for?""" + failures = [] + missing = [k for k in REQUIRED_KEYS if k not in analysis] + wrong_type = [k for k in LIST_KEYS if k in analysis and not isinstance(analysis[k], list)] + + score = 1.0 + if missing: + score -= min(0.8, 0.1 * len(missing)) + failures.append(f"format_compliance: missing keys ({', '.join(missing[:6])})") + if wrong_type: + score -= min(0.5, 0.15 * len(wrong_type)) + failures.append(f"format_compliance: non-list values for {', '.join(wrong_type[:4])}") + if not isinstance(analysis.get("summary"), str): + score -= 0.1 + failures.append("format_compliance: summary is not a string") + return max(0.0, score), failures + + +def evaluate_analysis( + analysis: Dict[str, Any], page_text: str, model: str = "" +) -> Dict[str, Any]: + """Score one page analysis across all five rubric dimensions.""" + scores: Dict[str, float] = {} + failures: List[str] = [] + + for name, scorer in ( + ("entity_accuracy", lambda: score_entity_accuracy(analysis, page_text)), + ("confidence_language", lambda: score_confidence_language(analysis)), + ("source_discipline", lambda: score_source_discipline(analysis, page_text)), + ("pivot_reasoning", lambda: score_pivot_reasoning(analysis)), + ("format_compliance", lambda: score_format_compliance(analysis)), + ): + value, dimension_failures = scorer() + scores[name] = round(max(0.0, min(1.0, value)), 4) + failures.extend(dimension_failures) + + weighted = sum(scores[k] * RUBRIC_WEIGHTS[k] for k in RUBRIC_WEIGHTS) + return { + "url": analysis.get("url", ""), + "model": model, + "scores": scores, + "weighted_score": round(weighted, 4), + "failures": failures, + "method": "deterministic_rubric", + } + + +def evaluate_run( + analyses: List[Dict[str, Any]], + text_loader: Callable[[str], str], + model: str = "", + weak_threshold: float = 0.6, +) -> CheckResult: + """Score every analysis in a run and rank the recurring failure modes.""" + result = CheckResult(check_name="Analysis Quality Evaluation") + if not analyses: + result.notes.append("No model analyses available. Evaluation skipped.") + return result + + evaluations: List[Dict[str, Any]] = [] + failure_counter: Counter = Counter() + + for analysis in analyses: + if not isinstance(analysis, dict) or analysis.get("error"): + continue + url = analysis.get("url") or "" + evaluation = evaluate_analysis(analysis, text_loader(url), model=model) + evaluations.append(evaluation) + for failure in evaluation["failures"]: + # Count the failure class, not the per-page numbers inside the message. + failure_counter[failure.split(":")[0].strip()] += 1 + + if not evaluations: + result.notes.append("No parseable model analyses to evaluate.") + return result + + weighted = [e["weighted_score"] for e in evaluations] + mean = sum(weighted) / len(weighted) + by_dimension = { + dimension: round( + sum(e["scores"][dimension] for e in evaluations) / len(evaluations), 4 + ) + for dimension in RUBRIC_WEIGHTS + } + + result.rows = evaluations + result.stats = { + "evaluated": len(evaluations), + "mean_weighted_score": round(mean, 4), + "min_weighted_score": round(min(weighted), 4), + "max_weighted_score": round(max(weighted), 4), + "by_dimension": by_dimension, + "dominant_failures": failure_counter.most_common(10), + "rubric_weights": RUBRIC_WEIGHTS, + } + + for dimension, score in sorted(by_dimension.items(), key=lambda kv: kv[1]): + if score >= weak_threshold: + continue + result.findings.append(Finding( + check="Analysis Quality", + item=dimension, + reason=( + f"Mean {dimension.replace('_', ' ')} across {len(evaluations)} analyses is " + f"{score:.2f}, below the {weak_threshold:.2f} threshold." + ), + next_step=( + "Treat model output on this dimension with added scepticism for this run, and " + "export weak examples for a separate, reviewed training workflow." + ), + origin=DERIVED, + priority=HIGH if score < 0.4 else MEDIUM, + sources=[e["url"] for e in evaluations if e["scores"][dimension] < weak_threshold][:25], + evidence={"dimension": dimension, "mean": score, "weight": RUBRIC_WEIGHTS[dimension]}, + method="deterministic_rubric", + model=model, + confidence=[deterministic_confidence(0.85, "computed against page text, not model-graded")], + )) + + result.notes.append( + f"Evaluated {len(evaluations)} analysis output(s). Mean weighted score {mean:.3f}. " + "Scoring is deterministic; no model graded another model." + ) + return result + + +def weak_examples(evaluation_result: CheckResult, threshold: float = 0.6) -> List[Dict[str, Any]]: + """Analyses that scored below threshold — the useful material for a training handoff.""" + return [ + row for row in evaluation_result.rows + if isinstance(row, dict) and row.get("weighted_score", 1.0) < threshold + ] diff --git a/src/osintai/extractor.py b/src/osintai/extractor.py index 1af08ce..b1945e1 100644 --- a/src/osintai/extractor.py +++ b/src/osintai/extractor.py @@ -3,6 +3,21 @@ from urllib.parse import urlparse from .normalize import absolutize + +def _validated_http_url(value: str) -> tuple[str, str] | None: + """Return a cleaned URL and hostname, or None for malformed regex matches.""" + candidate = value.rstrip(".,;:!?)}") + try: + parsed = urlparse(candidate) + hostname = parsed.hostname + except ValueError: + # urllib raises for malformed bracketed IPv6-like strings. + return None + if parsed.scheme not in {"http", "https"} or not hostname: + return None + return candidate, hostname.lower() + + class Extractor: EMAIL_RE = re.compile(r"\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\b") PHONE_RE = re.compile(r"\b\d{3}[-.]?\d{3}[-.]?\d{4}\b") @@ -46,19 +61,22 @@ def extract_indicators(self, url: str, text: str, html: str) -> dict: combined = "\n".join([text or "", html or ""]) emails = sorted(set(self.EMAIL_RE.findall(combined)))[:200] phones = sorted(set(self.PHONE_RE.findall(combined)))[:100] - urls = sorted(set(self.URL_RE.findall(combined)))[:100] + parsed_urls = { + parsed + for match in self.URL_RE.findall(combined) + if (parsed := _validated_http_url(match)) is not None + } + urls = sorted(url for url, _ in parsed_urls)[:100] btc_addresses = sorted(set(self.BTC_RE.findall(combined)))[:100] eth_addresses = sorted({addr.lower() for addr in self.ETH_RE.findall(combined)})[:100] social_handles = sorted(set(self.SOCIAL_RE.findall(combined)))[:100] ip_addresses = sorted(set(self.IP_RE.findall(combined)))[:100] - domain = urlparse(url).netloc + source = _validated_http_url(url) + domain = source[1] if source else "" domains = {domain} if domain else set() domains.update(self.DOMAIN_RE.findall(combined)) - for extracted_url in urls: - parsed = urlparse(extracted_url) - if parsed.netloc: - domains.add(parsed.netloc.lower()) + domains.update(hostname for _, hostname in parsed_urls) # Drop email-only host fragments that appear solely because of address parsing noise. email_domains = {email.rsplit("@", 1)[-1].lower() for email in emails if "@" in email} diff --git a/src/osintai/fetcher.py b/src/osintai/fetcher.py index 82b54a4..8b7e600 100644 --- a/src/osintai/fetcher.py +++ b/src/osintai/fetcher.py @@ -1,9 +1,23 @@ -import random import asyncio +import secrets +from collections.abc import Callable +from typing import Dict, Optional +from urllib.parse import urljoin + import httpx -from typing import Optional, Dict + from .proxy_pool import ProxyPool + +class FetchRejected(RuntimeError): + """Raised when a response violates the crawler's content safety policy.""" + + +REDIRECT_STATUSES = {301, 302, 303, 307, 308} +MAX_REDIRECTS = 10 +_RANDOM = secrets.SystemRandom() + + class AsyncFetcher: def __init__( self, @@ -12,7 +26,8 @@ def __init__( timeout_s: float = 20.0, min_delay_s: float = 0.2, max_delay_s: float = 1.2, - retries: int = 2 + retries: int = 2, + max_response_bytes: int = 2_000_000, ): self.user_agents = user_agents or ["Mozilla/5.0"] self.proxy_pool = proxy_pool @@ -20,9 +35,10 @@ def __init__( self.min_delay_s = min_delay_s self.max_delay_s = max_delay_s self.retries = retries + self.max_response_bytes = max(1, max_response_bytes) def _headers(self) -> Dict[str, str]: - ua = random.choice(self.user_agents) + ua = secrets.choice(self.user_agents) return { "User-Agent": ua, "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", @@ -32,8 +48,93 @@ def _headers(self) -> Dict[str, str]: "Upgrade-Insecure-Requests": "1", } - async def get(self, client: httpx.AsyncClient, url: str) -> httpx.Response: - await asyncio.sleep(random.uniform(self.min_delay_s, self.max_delay_s)) + async def _bounded_get( + self, + client: httpx.AsyncClient, + url: str, + request_kwargs: dict, + redirect_validator: Optional[Callable[[str], bool]] = None, + ) -> httpx.Response: + """Fetch HTML while validating every redirect before the next request.""" + current_url = url + for redirect_count in range(MAX_REDIRECTS + 1): + async with client.stream( + "GET", + current_url, + timeout=self.timeout_s, + follow_redirects=False, + **request_kwargs, + ) as resp: + if resp.status_code in REDIRECT_STATUSES: + location = resp.headers.get("location", "").strip() + if not location: + return self._empty_response(resp) + if redirect_count >= MAX_REDIRECTS: + raise FetchRejected(f"redirect limit exceeded ({MAX_REDIRECTS})") + next_url = urljoin(str(resp.url), location) + if redirect_validator is not None and not redirect_validator(next_url): + raise FetchRejected(f"redirect outside authorized scope: {next_url}") + current_url = next_url + continue + + if resp.status_code != 200: + # Status is all the crawler needs for unsuccessful responses; + # do not receive their response bodies. + return self._empty_response(resp) + + content_type = ( + resp.headers.get("content-type", "").partition(";")[0].strip().lower() + ) + if content_type not in {"text/html", "application/xhtml+xml"}: + raise FetchRejected( + f"content type {content_type or '(missing)'} is not HTML/XHTML" + ) + + content_length = resp.headers.get("content-length") + if content_length and content_length.isdigit(): + if int(content_length) > self.max_response_bytes: + raise FetchRejected( + f"response exceeds {self.max_response_bytes} byte limit" + ) + + content = bytearray() + async for chunk in resp.aiter_bytes(): + if len(content) + len(chunk) > self.max_response_bytes: + raise FetchRejected( + f"response exceeds {self.max_response_bytes} byte limit" + ) + content.extend(chunk) + + # aiter_bytes() returns decoded bytes; remove transport encodings so + # the reconstructed buffered response is not decoded a second time. + headers = dict(resp.headers) + headers.pop("content-encoding", None) + headers.pop("content-length", None) + return httpx.Response( + status_code=resp.status_code, + headers=headers, + content=bytes(content), + request=resp.request, + ) + + raise FetchRejected(f"redirect limit exceeded ({MAX_REDIRECTS})") + + @staticmethod + def _empty_response(resp: httpx.Response) -> httpx.Response: + return httpx.Response( + status_code=resp.status_code, + headers=dict(resp.headers), + content=b"", + request=resp.request, + ) + + async def get( + self, + client: httpx.AsyncClient, + url: str, + redirect_validator: Optional[Callable[[str], bool]] = None, + ) -> httpx.Response: + await asyncio.sleep(_RANDOM.uniform(self.min_delay_s, self.max_delay_s)) last_exc = None for _ in range(self.retries + 1): @@ -45,24 +146,25 @@ async def get(self, client: httpx.AsyncClient, url: str) -> httpx.Response: # httpx 0.28 configures proxies at client construction time. async with httpx.AsyncClient( proxy=proxy_url, - timeout=self.timeout_s, - follow_redirects=True, + trust_env=False, ) as proxy_client: - resp = await proxy_client.get(url, **request_kwargs) + resp = await self._bounded_get( + proxy_client, url, request_kwargs, redirect_validator + ) else: - resp = await client.get( - url, - timeout=self.timeout_s, - follow_redirects=True, - **request_kwargs, + resp = await self._bounded_get( + client, url, request_kwargs, redirect_validator ) if self.proxy_pool: self.proxy_pool.mark_ok(proxy_url) return resp + except FetchRejected: + # Policy failures are deterministic; retries only add traffic. + raise except Exception as e: last_exc = e if self.proxy_pool: self.proxy_pool.mark_fail(proxy_url) - await asyncio.sleep(0.25 + random.uniform(0.1, 0.4)) + await asyncio.sleep(0.25 + _RANDOM.uniform(0.1, 0.4)) raise last_exc diff --git a/src/osintai/hypotheses.py b/src/osintai/hypotheses.py new file mode 100644 index 0000000..e0b19f1 --- /dev/null +++ b/src/osintai/hypotheses.py @@ -0,0 +1,213 @@ +"""Hypothesis generation from deterministic findings. + +A hypothesis is an explanation that would account for what was observed. It is not a +finding, and OSINTai labels it as such at every point where it can be read: in the record, +in the artifact, and in the report. + +Every hypothesis here is generated deterministically from findings that already exist, and +each one carries what would confirm it and what would refute it. That last part is what +separates a usable investigative hypothesis from a suggestion — a hypothesis nobody can +settle is not worth writing down. + +Model-generated hypotheses are also supported (deep mode), and are labelled with the model +that produced them so the two kinds never blur together. +""" + +from __future__ import annotations + +from collections import defaultdict +from typing import Any, Dict, List, Optional + +from .entities import DOMAIN, EMAIL, EntityIndex, PHONE, USERNAME +from .provenance import ( + CROSS_MODEL, + HYPOTHESIS, + Confidence, + Finding, + Hypothesis, + MODEL_SELF, + deterministic_confidence, + model_self_confidence, + source_support, +) + + +def _by_check(findings: List[Finding]) -> Dict[str, List[Finding]]: + grouped: Dict[str, List[Finding]] = defaultdict(list) + for finding in findings: + grouped[finding.check].append(finding) + return grouped + + +def from_findings(findings: List[Finding], index: Optional[EntityIndex] = None) -> List[Hypothesis]: + """Derive candidate explanations from the deterministic finding set.""" + grouped = _by_check(findings) + hypotheses: List[Hypothesis] = [] + + homoglyphs = grouped.get("Unicode / Homoglyph", []) + for finding in homoglyphs: + ascii_form = finding.evidence.get("ascii_normalization", "") + hypotheses.append(Hypothesis( + statement=( + f"{finding.item!r} may be an impersonation of {ascii_form!r} rather than a " + "distinct legitimate identifier." + ), + supporting=[finding.reason], + contradicting=[ + "Non-ASCII characters are also normal for non-English organizations and personal names." + ], + sources=list(finding.sources), + would_confirm=( + "Registration or profile data showing the lookalike was created after the ASCII " + "form and by an unrelated party, or content copied from the ASCII form's site." + ), + would_refute=( + "Evidence that the same owner controls both forms, or that the identifier belongs " + "to a language that legitimately uses those characters." + ), + follow_up="Compare WHOIS, certificate history, and site content for both forms.", + method="homoglyph_finding", + confidence=[deterministic_confidence(0.45, "derived from one deterministic finding")], + )) + + secrets = grouped.get("Potential Secret Exposure", []) + grouped.get("Credential-Shaped Content", []) + if secrets: + sources = sorted({s for f in secrets for s in f.sources}) + hypotheses.append(Hypothesis( + statement=( + f"{len(sources)} crawled page(s) may be exposing live credential material through " + "misconfiguration or an unredacted publication." + ), + supporting=[f"{len(secrets)} deterministic secret/credential finding(s)."], + contradicting=[ + "Documentation, tutorials, and test fixtures routinely contain credential-shaped " + "placeholder values." + ], + sources=sources[:25], + would_confirm="The values resolve against a live service, or the host confirms an exposure.", + would_refute="The values are documented examples, expired, or randomly generated placeholders.", + follow_up=( + "Review the pages in context. If exposure is real, notify the owner; do not test " + "the credentials." + ), + method="secret_finding_cluster", + confidence=[deterministic_confidence(0.4, "pattern evidence only")], + )) + + correlations = grouped.get("Correlation Candidate", []) + strong = [f for f in correlations if (f.evidence or {}).get("score", 0) >= 0.7] + for finding in strong[:15]: + evidence = finding.evidence or {} + left = (evidence.get("left") or {}).get("value", "") + right = (evidence.get("right") or {}).get("value", "") + hypotheses.append(Hypothesis( + statement=f"{left!r} and {right!r} may be controlled by the same party.", + supporting=[finding.reason], + contradicting=[ + "Co-occurrence on a page frequently reflects a directory, aggregator, or comment " + "thread rather than shared control." + ], + sources=list(finding.sources), + would_confirm=( + "Both identifiers appearing together in registration data, an authored profile, or " + "a record the subject controls." + ), + would_refute="Either identifier resolving to a clearly unrelated owner.", + follow_up="Run the pivot list for both identifiers and compare the results.", + method=f"correlation:{evidence.get('relation', '')}", + confidence=[ + deterministic_confidence(float(evidence.get("score", 0.0)), "correlation candidate score"), + source_support(list(finding.sources)), + ], + )) + + generated = grouped.get("Generated-Content Fingerprint", []) + if len(generated) >= 3: + sources = sorted({s for f in generated for s in f.sources}) + hypotheses.append(Hypothesis( + statement=( + f"The crawled site may publish machine-generated content at scale " + f"({len(generated)} page(s) carry generation markers)." + ), + supporting=[f"{len(generated)} pages matched generated-text markers."], + contradicting=[ + "The markers can also appear in pages that quote or discuss model output." + ], + sources=sources[:25], + would_confirm="A sampling of pages showing uniform structure, timing, or byline patterns.", + would_refute="The flagged pages are articles about language models rather than output from them.", + follow_up="Sample the flagged pages before relying on the site as a primary source.", + method="generated_content_cluster", + confidence=[deterministic_confidence(0.5, f"{len(generated)} flagged pages")], + )) + + gaps = grouped.get("Activity Gap", []) + grouped.get("Open Trailing Gap", []) + for finding in gaps[:10]: + hypotheses.append(Hypothesis( + statement=( + f"The dated-content gap at {finding.item} may reflect activity that moved off the " + "crawled sources rather than activity that stopped." + ), + supporting=[finding.reason], + contradicting=[ + "The gap may equally reflect this crawl's depth and scope rather than the subject's behavior." + ], + sources=list(finding.sources), + would_confirm="Dated activity found on another channel inside the same window.", + would_refute="A wider crawl of the same sources filling the gap with content.", + follow_up="Widen depth or seed set across the gap window and re-run.", + method="temporal_gap", + confidence=[deterministic_confidence(0.35, "absence of evidence is not evidence of absence")], + )) + + return hypotheses + + +def from_model(payload: Dict[str, Any], model: str, sources: List[str]) -> List[Hypothesis]: + """Wrap model-proposed hypotheses so their origin is never lost. + + Anything the model returns arrives labelled HYPOTHESIS with the model recorded. Model + self-confidence is kept as its own signal and does not become factual reliability. + """ + items = payload.get("hypotheses") if isinstance(payload, dict) else None + if not isinstance(items, list): + return [] + + hypotheses: List[Hypothesis] = [] + for item in items[:25]: + if isinstance(item, str): + statement, supporting, contradicting = item, [], [] + confirm = refute = follow_up = "" + self_confidence = None + elif isinstance(item, dict): + statement = str(item.get("statement") or item.get("hypothesis") or "").strip() + supporting = [str(s) for s in (item.get("supporting") or [])][:10] + contradicting = [str(s) for s in (item.get("contradicting") or [])][:10] + confirm = str(item.get("would_confirm") or "") + refute = str(item.get("would_refute") or "") + follow_up = str(item.get("follow_up") or item.get("recommended_follow_up") or "") + self_confidence = item.get("confidence") + else: + continue + + if not statement: + continue + + confidences = [] + if isinstance(self_confidence, (int, float)): + value = float(self_confidence) + confidences.append(model_self_confidence(value / 10.0 if value > 1 else value, model)) + + hypotheses.append(Hypothesis( + statement=statement, + supporting=supporting, + contradicting=contradicting or ["Not independently corroborated by deterministic analysis."], + sources=sources[:25], + would_confirm=confirm or "Not specified by the model.", + would_refute=refute or "Not specified by the model.", + follow_up=follow_up, + method="model_deep_analysis", + model=model, + confidence=confidences, + )) + return hypotheses diff --git a/src/osintai/multimodel.py b/src/osintai/multimodel.py new file mode 100644 index 0000000..b51dd62 --- /dev/null +++ b/src/osintai/multimodel.py @@ -0,0 +1,283 @@ +"""Optional multi-model cross-checking and disagreement handling. + +None of this runs unless the operator asks for it. OSINTai's default remains one model, one +call per page; expensive analysis that runs whether or not anyone wanted it is how a fast +tool stops being one. + +Where models disagree, the disagreement is preserved. It is not averaged away, and a +majority does not silently become the answer. Two competent models reaching opposite +conclusions about a claim is itself a finding, and usually a more useful one than either +verdict alone. +""" + +from __future__ import annotations + +import asyncio +from collections import Counter +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional, Tuple + +from .prompts import cross_check_prompt +from .provenance import ( + HIGH, + LOW, + MEDIUM, + MODEL, + CheckResult, + Confidence, + Finding, + cross_model_confidence, + model_self_confidence, +) + +SUPPORTED = "supported" +UNSUPPORTED = "unsupported" +CONTRADICTED = "contradicted" +INSUFFICIENT = "insufficient_evidence" + +VERDICTS = (SUPPORTED, UNSUPPORTED, CONTRADICTED, INSUFFICIENT) + +# Verdicts that genuinely conflict, as opposed to merely differing in strength. +_OPPOSED = {(SUPPORTED, CONTRADICTED), (CONTRADICTED, SUPPORTED), + (SUPPORTED, UNSUPPORTED), (UNSUPPORTED, SUPPORTED)} + + +@dataclass +class ModelVerdict: + model: str + verdict: str + reason: str = "" + confidence: float = 0.0 + error: str = "" + + def to_dict(self) -> Dict[str, Any]: + return { + "model": self.model, + "verdict": self.verdict, + "reason": self.reason, + "self_confidence": round(self.confidence, 4), + "error": self.error, + } + + +@dataclass +class CrossCheck: + """One claim assessed by several models, with agreement state preserved.""" + + claim: str + sources: List[str] = field(default_factory=list) + verdicts: List[ModelVerdict] = field(default_factory=list) + + @property + def answered(self) -> List[ModelVerdict]: + return [v for v in self.verdicts if not v.error and v.verdict in VERDICTS] + + @property + def agreement(self) -> str: + answered = self.answered + if len(answered) < 2: + return "not_cross_checked" + distinct = {v.verdict for v in answered} + if len(distinct) == 1: + return "unanimous" + for left in distinct: + for right in distinct: + if (left, right) in _OPPOSED: + return "contradictory" + return "divided" + + @property + def majority(self) -> Optional[str]: + """The most common verdict, or None when models are evenly split. + + Reported as a count, never as the answer. + """ + answered = self.answered + if not answered: + return None + counts = Counter(v.verdict for v in answered).most_common() + if len(counts) > 1 and counts[0][1] == counts[1][1]: + return None + return counts[0][0] + + def to_dict(self) -> Dict[str, Any]: + answered = self.answered + agreeing = sum(1 for v in answered if v.verdict == self.majority) if self.majority else 0 + return { + "claim": self.claim, + "sources": self.sources, + "agreement": self.agreement, + "majority_verdict": self.majority, + "agreeing": agreeing, + "answered": len(answered), + "requested": len(self.verdicts), + "verdicts": [v.to_dict() for v in self.verdicts], + } + + +def _parse_verdict(model: str, payload: Optional[Dict[str, Any]]) -> ModelVerdict: + if not isinstance(payload, dict): + return ModelVerdict(model=model, verdict="", error="no parseable response") + verdict = str(payload.get("verdict", "")).strip().lower().replace(" ", "_") + if verdict not in VERDICTS: + return ModelVerdict(model=model, verdict="", error=f"unrecognized verdict {verdict!r}") + raw_confidence = payload.get("confidence", 0.0) + try: + confidence = float(raw_confidence) + except (TypeError, ValueError): + confidence = 0.0 + if confidence > 1: + confidence = confidence / 10.0 + return ModelVerdict( + model=model, + verdict=verdict, + reason=str(payload.get("reason", ""))[:400], + confidence=max(0.0, min(1.0, confidence)), + ) + + +async def cross_check_claims( + ollama, + models: List[str], + claims: List[Tuple[str, str, List[str]]], + timeout_s: float = 90.0, + max_parallel: int = 2, +) -> List[CrossCheck]: + """Put each claim to every model. `claims` is (claim, context, sources). + + Model failure is tolerated per verdict: one unreachable model leaves its slot marked with + an error and the remaining verdicts still form a cross-check. + """ + if not models or not claims: + return [] + + semaphore = asyncio.Semaphore(max(1, max_parallel)) + results: List[CrossCheck] = [] + + async def ask(model: str, prompt: str) -> ModelVerdict: + async with semaphore: + try: + payload = await ollama.async_generate_json(model, prompt, timeout_s) + except Exception as exc: + return ModelVerdict(model=model, verdict="", error=str(exc)[:200]) + return _parse_verdict(model, payload) + + for claim, context, sources in claims: + prompt = cross_check_prompt(claim, context) + verdicts = await asyncio.gather(*[ask(model, prompt) for model in models]) + results.append(CrossCheck(claim=claim, sources=list(sources), verdicts=list(verdicts))) + + return results + + +def summarize_cross_checks(checks: List[CrossCheck]) -> CheckResult: + """Turn cross-check outcomes into findings, keeping disagreement visible.""" + result = CheckResult(check_name="Multi-Model Cross-Check") + if not checks: + result.notes.append("Multi-model cross-check not requested.") + return result + + contradictory = 0 + for check in checks: + payload = check.to_dict() + result.rows.append(payload) + answered = check.answered + agreement = check.agreement + + if agreement == "not_cross_checked": + result.errors.append( + f"Claim cross-check incomplete ({len(answered)}/{len(check.verdicts)} models answered): " + f"{check.claim[:100]}" + ) + continue + + if agreement == "contradictory": + contradictory += 1 + positions = "; ".join( + f"{v.model} says {v.verdict}" + (f" ({v.reason[:80]})" if v.reason else "") + for v in answered + ) + result.findings.append(Finding( + check="Model Disagreement", + item=check.claim[:200], + reason=f"Models reached opposing verdicts. {positions}", + next_step=( + "Resolve this against the source material directly. The disagreement is " + "recorded unresolved; no model's verdict has been adopted." + ), + origin=MODEL, + priority=HIGH, + sources=check.sources, + evidence=payload, + method="multi_model_cross_check", + model=", ".join(v.model for v in answered), + confidence=[ + cross_model_confidence(0, len(answered)), + *[model_self_confidence(v.confidence, v.model) for v in answered], + ], + )) + continue + + agreeing = sum(1 for v in answered if v.verdict == check.majority) + priority = LOW if agreement == "unanimous" else MEDIUM + result.findings.append(Finding( + check="Cross-Model Assessment", + item=check.claim[:200], + reason=( + f"{agreeing}/{len(answered)} models returned {check.majority!r}" + + (" (unanimous)." if agreement == "unanimous" else " (divided).") + ), + next_step=( + "Agreement between models indicates consistency, not verification. Confirm " + "against the source before relying on it." + ), + origin=MODEL, + priority=priority, + sources=check.sources, + evidence=payload, + method="multi_model_cross_check", + model=", ".join(v.model for v in answered), + confidence=[ + cross_model_confidence(agreeing, len(answered)), + *[model_self_confidence(v.confidence, v.model) for v in answered], + ], + )) + + result.stats = { + "claims_checked": len(checks), + "contradictory": contradictory, + "unanimous": sum(1 for c in checks if c.agreement == "unanimous"), + "divided": sum(1 for c in checks if c.agreement == "divided"), + } + result.notes.append( + f"Cross-checked {len(checks)} claim(s). {contradictory} produced contradictory verdicts, " + "which are preserved rather than resolved." + ) + return result + + +def claims_from_analyses( + analyses: List[Dict[str, Any]], limit: int = 12 +) -> List[Tuple[str, str, List[str]]]: + """Pick the model claims most worth a second opinion. + + Risk flags first: they are the highest-weighted input to the page score, so an + unsupported one distorts the ranking more than anything else the model says. + """ + claims: List[Tuple[str, str, List[str]]] = [] + for analysis in analyses: + if not isinstance(analysis, dict): + continue + url = analysis.get("url") or "" + summary = str(analysis.get("summary") or "") + for flag in (analysis.get("risk_flags") or [])[:3]: + if not isinstance(flag, str) or not flag.strip(): + continue + claims.append(( + f"The page at {url} supports this risk assessment: {flag}", + f"Page summary: {summary}", + [url], + )) + if len(claims) >= limit: + return claims + return claims diff --git a/src/osintai/normalize.py b/src/osintai/normalize.py index 3b0e314..05e31f2 100644 --- a/src/osintai/normalize.py +++ b/src/osintai/normalize.py @@ -18,7 +18,7 @@ def absolutize(base_url: str, href: str) -> str: return None try: return urljoin(base_url, href) - except: + except ValueError: return None def same_domain(url1: str, url2: str) -> bool: @@ -27,5 +27,5 @@ def same_domain(url1: str, url2: str) -> bool: d1 = urlparse(url1).netloc.lower() d2 = urlparse(url2).netloc.lower() return d1 == d2 - except: + except ValueError: return False diff --git a/src/osintai/ollama_api.py b/src/osintai/ollama_api.py index 624314d..df61695 100644 --- a/src/osintai/ollama_api.py +++ b/src/osintai/ollama_api.py @@ -1,13 +1,36 @@ import json +import ipaddress from typing import Optional, Dict, Any, List +from urllib.parse import urlsplit import httpx + +def _validate_local_base_url(value: str) -> str: + """Accept only credential-free loopback Ollama endpoints.""" + parsed = urlsplit((value or "").strip()) + if parsed.scheme not in {"http", "https"} or not parsed.hostname: + raise ValueError("Ollama endpoint must be an HTTP(S) loopback URL") + if parsed.username or parsed.password or parsed.query or parsed.fragment: + raise ValueError("Ollama endpoint cannot contain credentials, a query, or a fragment") + if parsed.path not in {"", "/"}: + raise ValueError("Ollama endpoint must not contain a path") + host = parsed.hostname.lower() + if host != "localhost": + try: + if not ipaddress.ip_address(host).is_loopback: + raise ValueError("Ollama endpoint must resolve to a loopback address") + except ValueError as exc: + if "loopback" in str(exc): + raise + raise ValueError("Ollama endpoint must resolve to a loopback address") from exc + return value.rstrip("/") + class OllamaAPI: """Ollama API client for LLM analysis and embeddings.""" def __init__(self, base_url: str = "http://localhost:11434"): - self.base_url = base_url.rstrip("/") + self.base_url = _validate_local_base_url(base_url) def generate_json(self, model: str, prompt: str, timeout_s: float = 60.0) -> Optional[Dict[str, Any]]: """Generate JSON response from model.""" @@ -28,7 +51,7 @@ async def async_generate_json(self, model: str, prompt: str, timeout_s: float = } } try: - async with httpx.AsyncClient(timeout=timeout_s) as client: + async with httpx.AsyncClient(timeout=timeout_s, trust_env=False) as client: r = await client.post(url, json=payload) r.raise_for_status() data = r.json() @@ -46,7 +69,7 @@ async def async_embed(self, model: str, input_text: str, timeout_s: float = 30.0 url = f"{self.base_url}/api/embeddings" payload = {"model": model, "prompt": input_text} try: - async with httpx.AsyncClient(timeout=timeout_s) as client: + async with httpx.AsyncClient(timeout=timeout_s, trust_env=False) as client: r = await client.post(url, json=payload) r.raise_for_status() data = r.json() @@ -55,6 +78,53 @@ async def async_embed(self, model: str, input_text: str, timeout_s: float = 30.0 except Exception: return None + async def async_generate_text(self, model: str, prompt: str, timeout_s: float = 60.0) -> Optional[str]: + """Free-text generation. Same endpoint and options as generate_json, without format=json.""" + url = f"{self.base_url}/api/generate" + payload = { + "model": model, + "prompt": prompt, + "stream": False, + "options": { + "temperature": 0.2, + "top_p": 0.8, + "num_ctx": 8192 + } + } + try: + async with httpx.AsyncClient(timeout=timeout_s, trust_env=False) as client: + r = await client.post(url, json=payload) + r.raise_for_status() + return r.json().get("response", "") + except Exception: + return None + + def health(self, timeout_s: float = 5.0) -> Dict[str, Any]: + """Is the local model server reachable, and what does it have loaded? + + Reported at startup so an unreachable server is visible before the crawl rather + than showing up as pages that silently produced no analysis. + """ + try: + with httpx.Client(timeout=timeout_s, trust_env=False) as client: + r = client.get(f"{self.base_url}/api/tags") + r.raise_for_status() + models = [m.get("name", "") for m in r.json().get("models", []) if isinstance(m, dict)] + return {"reachable": True, "models": sorted(n for n in models if n), "error": ""} + except Exception as exc: + return {"reachable": False, "models": [], "error": str(exc)} + + def list_models(self, timeout_s: float = 5.0) -> List[str]: + return self.health(timeout_s)["models"] + + def has_model(self, model: str, timeout_s: float = 5.0) -> bool: + """Exact name first, then the bare name, since ':latest' is often left off.""" + available = self.list_models(timeout_s) + if model in available: + return True + base = model.split(":")[0] + return any(name.split(":")[0] == base for name in available) + def analyze(self, url: str, title: str, text: str, model: str = "osint-tuned-v3:latest") -> Optional[Dict[str, Any]]: """Analyze content for intelligence value.""" prompt = f"""Analyze this webpage for OSINT intelligence value. diff --git a/src/osintai/patterns.py b/src/osintai/patterns.py new file mode 100644 index 0000000..96ea45b --- /dev/null +++ b/src/osintai/patterns.py @@ -0,0 +1,506 @@ +"""Deterministic pattern and anomaly checks over crawled material. + +Everything here is pure code. No model is consulted, nothing is inferred, and every finding +can be reproduced from the saved run artifacts. That is the point: OSINTai's only analysis +before this was whatever the model chose to say, and a finding an operator cannot reproduce +is a finding they cannot testify to. + +The checks cover homoglyph and zero-width characters, sensitive domains, internal IPs, +secret-shaped material, JWT structure, credential-shaped text, and generated-text markers. + +Matched secret values are never written into findings. The finding records that +credential-shaped material exists, where, and how much. +""" + +from __future__ import annotations + +import base64 +import binascii +import json +import math +import unicodedata +from collections import Counter +from typing import Any, Dict, Iterable, List, Optional + +from .provenance import ( + DERIVED, + HIGH, + LOW, + MEDIUM, + OBSERVED, + CheckResult, + Finding, + deterministic_confidence, + source_support, +) + +ZERO_WIDTH = { + "​": "ZERO WIDTH SPACE", + "‌": "ZERO WIDTH NON-JOINER", + "‍": "ZERO WIDTH JOINER", + "⁠": "WORD JOINER", + "": "ZERO WIDTH NO-BREAK SPACE", + "­": "SOFT HYPHEN", +} + +# Practical investigative mapping of characters that render as ASCII but are not. +HOMOGLYPH_MAP: Dict[str, str] = { + "а": "a", "Α": "A", "А": "A", "ɑ": "a", "à": "a", "á": "a", "ä": "a", + "е": "e", "Ε": "E", "Е": "E", "è": "e", "é": "e", "ë": "e", + "і": "i", "Ι": "I", "І": "I", "í": "i", "ï": "i", "ı": "i", + "ο": "o", "О": "O", "Ο": "O", "о": "o", "ò": "o", "ó": "o", "ö": "o", + "р": "p", "Ρ": "P", "Р": "P", + "с": "c", "С": "C", "Ϲ": "C", + "х": "x", "Χ": "X", "Х": "X", + "у": "y", "Υ": "Y", "Ү": "Y", + "ѕ": "s", "Ѕ": "S", + "ԁ": "d", "Ԁ": "D", + "Ь": "b", "Ꮟ": "b", + "ӏ": "l", "ⅼ": "l", "Ⅰ": "I", + "ｍ": "m", "ｎ": "n", "ａ": "a", "ｂ": "b", "ｃ": "c", "ｅ": "e", "ｉ": "i", + "ｏ": "o", "ｐ": "p", "ｓ": "s", "ｔ": "t", "ｕ": "u", "ｘ": "x", "ｙ": "y", +} + +SENSITIVE_TLDS = (".gov", ".mil") + +# Phrases that leak through when generated text is published without editing. Cheap, +# deterministic, and a real source-validation signal on a modern crawl. +AI_TEXT_MARKERS = ( + "as an ai language model", + "as an ai assistant", + "i'm sorry, but i cannot", + "i cannot fulfill that request", + "my training data", + "i do not have personal opinions", + "knowledge cutoff", + "as a large language model", + "i'm unable to provide", + "regenerate response", +) + + +def _codepoints(text: str) -> str: + return " ".join(f"U+{ord(ch):04X}" for ch in text) + + +def _char_name(ch: str) -> str: + if ch in ZERO_WIDTH: + return ZERO_WIDTH[ch] + return unicodedata.name(ch, "UNKNOWN") + + +def _is_private_ip(ip: str) -> bool: + parts = ip.split(".") + if len(parts) != 4: + return False + try: + octets = [int(p) for p in parts] + except ValueError: + return False + if any(o < 0 or o > 255 for o in octets): + return False + a, b = octets[0], octets[1] + if a == 10: + return True + if a == 192 and b == 168: + return True + if a == 172 and 16 <= b <= 31: + return True + if a == 127: + return True + if a == 169 and b == 254: + return True + return False + + +def _shannon_entropy(value: str) -> float: + if not value: + return 0.0 + counts = Counter(value) + length = len(value) + return -sum((c / length) * math.log2(c / length) for c in counts.values()) + + +def decode_jwt_payload(token: str) -> Optional[Dict[str, Any]]: + """Decode a JWT's claim set. Structure only; no signature is verified.""" + parts = token.split(".") + if len(parts) != 3: + return None + payload = parts[1] + padding = "=" * (-len(payload) % 4) + try: + raw = base64.urlsafe_b64decode(payload + padding) + decoded = json.loads(raw.decode("utf-8", errors="ignore")) + except (binascii.Error, ValueError, UnicodeDecodeError): + return None + return decoded if isinstance(decoded, dict) else None + + +def check_homoglyphs(index) -> CheckResult: + """Non-ASCII lookalikes in crawled domains and handles. + + A domain that reads as a familiar brand but carries a Cyrillic character is one of the + higher-value things a crawl can surface, and OSINTai could not see it before. + """ + from .entities import DOMAIN, USERNAME + + result = CheckResult(check_name="Unicode / Homoglyph Analysis") + candidates = index.of_kind(DOMAIN) + index.of_kind(USERNAME) + if not candidates: + result.notes.append("No domains or handles collected. Homoglyph analysis skipped.") + return result + + for entity in candidates: + value = entity.value + flagged: List[str] = [] + mapped: List[str] = [] + has_zero_width = False + for ch in value: + cp = ord(ch) + if 0x20 <= cp <= 0x7E: + continue + if ch in ZERO_WIDTH: + has_zero_width = True + flagged.append(f"{ch} U+{cp:04X} {_char_name(ch)}") + replacement = HOMOGLYPH_MAP.get(ch) + mapped.append( + f"U+{cp:04X} -> {replacement}" if replacement else f"U+{cp:04X} -> no ASCII mapping" + ) + + if not flagged: + continue + + ascii_form = "".join(HOMOGLYPH_MAP.get(ch, ch) for ch in value) + priority = HIGH if has_zero_width else MEDIUM + reason = ( + "Contains zero-width control characters, which are invisible when rendered." + if has_zero_width + else "Contains non-ASCII characters that render similarly to ASCII." + ) + result.rows.append({ + "value": value, + "kind": entity.kind, + "codepoints": _codepoints(value), + "flagged_characters": "; ".join(flagged), + "ascii_normalization": ascii_form, + "sources": len(entity.sources), + }) + result.findings.append(Finding( + check="Unicode / Homoglyph", + item=value, + reason=reason, + next_step=( + "Compare against the canonical ASCII spelling " + f"({ascii_form!r}), check registration and certificate history for both forms, " + "and confirm which one the subject actually controls." + ), + origin=DERIVED, + priority=priority, + sources=list(entity.sources), + evidence={ + "codepoints": _codepoints(value), + "flagged_characters": flagged, + "homoglyph_mapping": mapped, + "ascii_normalization": ascii_form, + }, + method="unicode_codepoint_scan", + confidence=[ + deterministic_confidence(1.0, "codepoint inspection is exact"), + source_support(list(entity.sources)), + ], + )) + + result.notes.append( + f"Inspected {len(candidates)} domain(s)/handle(s). " + f"Flagged {result.finding_count} containing non-ASCII characters." + ) + return result + + +def check_sensitive_infrastructure(index) -> CheckResult: + """Government/military identifiers and non-routable addresses in crawled content.""" + from .entities import DOMAIN, EMAIL, IP + + result = CheckResult(check_name="Sensitive Infrastructure Indicators") + + for entity in index.of_kind(EMAIL): + domain_part = entity.canonical_value.rsplit("@", 1)[-1] + if domain_part.endswith(SENSITIVE_TLDS): + result.findings.append(Finding( + check="Sensitive Email Domain", + item=entity.value, + reason=f"Email address on a restricted-use domain ({domain_part}).", + next_step=( + "Confirm the address is genuinely published rather than scraped or spoofed, " + "and handle under the rules applying to government contact data." + ), + origin=OBSERVED, + priority=HIGH, + sources=list(entity.sources), + evidence={"domain": domain_part}, + method="tld_rule", + confidence=[deterministic_confidence(1.0, "suffix match"), + source_support(list(entity.sources))], + )) + + for entity in index.of_kind(DOMAIN): + if entity.canonical_value.endswith(SENSITIVE_TLDS): + result.findings.append(Finding( + check="Sensitive Domain", + item=entity.value, + reason="Domain on a restricted-use top-level domain.", + next_step="Record the referencing page and confirm whether the reference is authentic.", + origin=OBSERVED, + priority=MEDIUM, + sources=list(entity.sources), + evidence={"domain": entity.canonical_value}, + method="tld_rule", + confidence=[deterministic_confidence(1.0, "suffix match"), + source_support(list(entity.sources))], + )) + + for entity in index.of_kind(IP): + if _is_private_ip(entity.value): + result.findings.append(Finding( + check="Internal IP Exposure", + item=entity.value, + reason="Non-routable (internal) IP address published on a public page.", + next_step=( + "Check whether the page leaks internal network topology, error output, or " + "configuration; capture the surrounding context before it changes." + ), + origin=OBSERVED, + priority=MEDIUM, + sources=list(entity.sources), + evidence={"ip": entity.value}, + method="rfc1918_rule", + confidence=[deterministic_confidence(1.0, "address range check"), + source_support(list(entity.sources))], + )) + + result.rows = [ + {"item": f.item, "check": f.check, "sources": len(f.sources)} for f in result.findings + ] + result.notes.append(f"Flagged {result.finding_count} sensitive infrastructure indicator(s).") + return result + + +def check_secret_exposure(page_extras: Dict[str, Dict[str, List[str]]]) -> CheckResult: + """Credential-shaped and secret-shaped material on crawled pages. + + Deliberately reports presence and location only. The matched value is not copied into + the finding, the evidence dict, or the report. + """ + result = CheckResult(check_name="Credential and Secret Exposure") + + for url, extras in (page_extras or {}).items(): + tokens = extras.get("api_tokens") or [] + jwts = extras.get("jwts") or [] + pairs = extras.get("credential_pairs") or [] + + if tokens: + high_entropy = [t for t in tokens if _shannon_entropy(t) >= 3.5] + result.findings.append(Finding( + check="Potential Secret Exposure", + item=url, + reason=( + f"{len(tokens)} key/token-shaped value(s) found in page content, " + f"{len(high_entropy)} of them high-entropy." + ), + next_step=( + "Open the page and confirm whether these are live credentials, example " + "placeholders, or public identifiers. If live, treat as an exposure and " + "notify the owner rather than using them." + ), + origin=DERIVED, + priority=HIGH if high_entropy else MEDIUM, + sources=[url], + evidence={ + "token_count": len(tokens), + "high_entropy_count": len(high_entropy), + "value_recorded": False, + }, + method="token_pattern_entropy", + confidence=[deterministic_confidence( + 0.6, "pattern match; placeholders and examples also match")], + )) + + for token in jwts: + claims = decode_jwt_payload(token) + claim_keys = sorted(claims.keys()) if claims else [] + result.findings.append(Finding( + check="JWT Present", + item=url, + reason=( + "JSON Web Token found in page content" + + (f" carrying claims: {', '.join(claim_keys[:12])}." if claim_keys else ".") + ), + next_step=( + "Review the decoded claim set for issuer, subject, audience and expiry to " + "identify the issuing system. Do not replay the token." + ), + origin=DERIVED, + priority=HIGH if claims else MEDIUM, + sources=[url], + evidence={ + "claim_keys": claim_keys, + "issuer": str(claims.get("iss", "")) if claims else "", + "audience": str(claims.get("aud", "")) if claims else "", + "expiry": str(claims.get("exp", "")) if claims else "", + "recorded_value": False, + }, + method="jwt_structural_decode", + confidence=[deterministic_confidence( + 0.95 if claims else 0.5, + "three-segment structure decoded" if claims else "structure matched, payload unreadable")], + )) + + if pairs: + result.findings.append(Finding( + check="Credential-Shaped Content", + item=url, + reason=f"{len(pairs)} line(s) matching an identifier:secret layout.", + next_step=( + "Determine whether the page is a leak, a configuration sample, or unrelated " + "colon-delimited data. Preserve the page capture before it is removed." + ), + origin=DERIVED, + priority=HIGH, + sources=[url], + evidence={"pair_count": len(pairs), "values_recorded": False}, + method="credential_pattern", + confidence=[deterministic_confidence( + 0.5, "layout match only; colon-delimited data is common")], + )) + + result.rows = [ + {"url": f.item, "check": f.check, "priority": f.priority} for f in result.findings + ] + result.notes.append( + f"Flagged {result.finding_count} secret/credential exposure indicator(s). " + "Matched values are intentionally not recorded." + ) + return result + + +def check_generated_text(pages: Iterable[Dict[str, Any]], text_loader) -> CheckResult: + """Pages carrying unedited language-model output. + + Source validation: content that was generated rather than written changes what a page is + worth as evidence, and it is worth knowing before the page is cited. + """ + result = CheckResult(check_name="Generated-Content Fingerprint") + checked = 0 + + for page in pages or []: + url = page.get("url") + if not url: + continue + text = text_loader(url) + if not text: + continue + checked += 1 + lowered = text.lower() + hits = [marker for marker in AI_TEXT_MARKERS if marker in lowered] + if not hits: + continue + result.rows.append({"url": url, "markers": "; ".join(hits), "marker_count": len(hits)}) + result.findings.append(Finding( + check="Generated-Content Fingerprint", + item=url, + reason=( + f"Page contains {len(hits)} phrase(s) characteristic of unedited language-model " + "output." + ), + next_step=( + "Weigh this page lower as a primary source. Check whether the site publishes " + "generated content at scale before relying on any claim it makes." + ), + origin=DERIVED, + priority=MEDIUM if len(hits) == 1 else HIGH, + sources=[url], + evidence={"markers": hits}, + method="phrase_fingerprint", + confidence=[deterministic_confidence( + min(0.9, 0.45 + 0.15 * len(hits)), + f"{len(hits)} marker phrase(s) matched")], + )) + + result.notes.append( + f"Scanned {checked} page text(s). Flagged {result.finding_count} with generated-content markers." + ) + return result + + +def check_recurring_and_outliers(index, page_scores: List[Dict[str, Any]]) -> CheckResult: + """Recurring cross-page signals and statistical outliers in page scores. + + Both are stated as what they are: a distribution observation, not a conclusion. A page + two standard deviations above the mean is unusual for this run and nothing more. + """ + result = CheckResult(check_name="Recurring Signals and Outliers") + + recurring = index.multi_source(minimum=3)[:40] + for entity in recurring: + result.rows.append({ + "kind": entity.kind, + "value": entity.value, + "source_count": len(entity.sources), + }) + result.findings.append(Finding( + check="Recurring Signal", + item=entity.value, + reason=f"Appears on {len(entity.sources)} independently crawled pages.", + next_step=( + "Treat as a spine for the investigation: establish who controls this identifier " + "and why it links these pages." + ), + origin=DERIVED, + priority=MEDIUM, + sources=list(entity.sources)[:25], + evidence={"kind": entity.kind, "source_count": len(entity.sources)}, + method="cross_page_frequency", + confidence=[source_support(list(entity.sources))], + )) + + scores = [float(p.get("score") or 0.0) for p in page_scores or []] + if len(scores) >= 5: + mean = sum(scores) / len(scores) + variance = sum((s - mean) ** 2 for s in scores) / len(scores) + stdev = math.sqrt(variance) + result.stats = { + "page_count": len(scores), + "score_mean": round(mean, 3), + "score_stdev": round(stdev, 3), + "score_max": round(max(scores), 3), + } + if stdev > 0: + threshold = mean + 2 * stdev + for page in page_scores: + score = float(page.get("score") or 0.0) + if score < threshold: + continue + result.findings.append(Finding( + check="Score Outlier", + item=page.get("url", ""), + reason=( + f"Page signal score {score:.2f} is more than two standard deviations " + f"above this run's mean of {mean:.2f}." + ), + next_step="Review this page first; it carries the densest indicator set in the run.", + origin=DERIVED, + priority=MEDIUM, + sources=[page.get("url", "")], + evidence={"score": score, "mean": round(mean, 3), "stdev": round(stdev, 3)}, + method="two_sigma_outlier", + confidence=[deterministic_confidence( + 0.7, "distribution statistic over this run only")], + )) + else: + result.notes.append("Fewer than 5 scored pages; outlier statistics not computed.") + + result.notes.append( + f"Identified {len(recurring)} recurring signal(s) and " + f"{sum(1 for f in result.findings if f.check == 'Score Outlier')} score outlier(s)." + ) + return result diff --git a/src/osintai/pipeline.py b/src/osintai/pipeline.py new file mode 100644 index 0000000..0bd823e --- /dev/null +++ b/src/osintai/pipeline.py @@ -0,0 +1,525 @@ +"""The OSINTai analysis lifecycle. + +Runs after the crawl, over what the crawl already saved. The crawl itself is untouched: the +per-page fetch, extract, analyze, score path is exactly what it always was, and this stage +reads its output rather than changing it. + + RUN ARTIFACTS + | + NORMALIZE / INDEX ENTITIES + | + DETERMINISTIC ANALYSIS (patterns, correlation, temporal) + | + MODEL-ASSISTED ANALYSIS (optional: deep analysis, cross-check) + | + CONFIDENCE / SUPPORT + | + HYPOTHESES / LEADS + | + REPORT + +Each stage collects its problems into notes and errors instead of raising, and the +orchestrator wraps every stage so one failure cannot end the run. An analysis stage failing +must never cost the operator the crawl they just paid for. +""" + +from __future__ import annotations + +import json +import os +import time +from dataclasses import dataclass, field +from typing import Any, Callable, Dict, List, Optional, Tuple + +from . import correlation as correlation_module +from . import evaluation as evaluation_module +from . import hypotheses as hypotheses_module +from . import multimodel +from . import patterns +from . import pivots +from . import temporal +from . import training_export as training_export_module +from .entities import ( + EntityIndex, + extract_extended, + index_indicators, +) +from .prompts import STANDARD, deep_analysis_prompt, page_prompt +from .provenance import ( + CheckResult, + Finding, + Hypothesis, + Lead, + sort_findings, +) +from .storage import append_jsonl, safe_mkdir, sha1, write_json + +# Cap on page texts read back for extended extraction. A very large crawl should not turn +# the analysis stage into a second crawl-length operation. +MAX_TEXT_PAGES = 2000 + + +@dataclass +class AnalysisOptions: + """Everything the analysis stage can be asked to do. Defaults are the fast path.""" + + deep: bool = False + cross_check_models: List[str] = field(default_factory=list) + evaluate: bool = False + training_export: bool = False + gap_days: int = 90 + prompt_profile: str = STANDARD + model: str = "" + use_ollama: bool = True + experimental_recursive: int = 0 + max_leads_per_kind: int = 10 + + +@dataclass +class AnalysisOutput: + results: List[CheckResult] = field(default_factory=list) + findings: List[Finding] = field(default_factory=list) + hypotheses: List[Hypothesis] = field(default_factory=list) + leads: List[Lead] = field(default_factory=list) + artifacts: Dict[str, str] = field(default_factory=dict) + stats: Dict[str, Any] = field(default_factory=dict) + errors: List[str] = field(default_factory=list) + + +def read_jsonl(path: str) -> List[Dict[str, Any]]: + if not os.path.exists(path): + return [] + rows: List[Dict[str, Any]] = [] + with open(path, "r", encoding="utf-8", errors="ignore") as handle: + for line in handle: + line = line.strip() + if not line: + continue + try: + rows.append(json.loads(line)) + except ValueError: + continue + return rows + + +class RunArtifacts: + """Reader for one run directory. Loads lazily and caches page text.""" + + def __init__(self, run_dir: str): + self.run_dir = run_dir + self.text_dir = os.path.join(run_dir, "pages_text") + self.analysis_dir = os.path.join(run_dir, "analysis") + self._text_cache: Dict[str, str] = {} + + def indicators(self) -> List[Dict[str, Any]]: + return read_jsonl(os.path.join(self.run_dir, "indicators.jsonl")) + + def page_scores(self) -> List[Dict[str, Any]]: + return read_jsonl(os.path.join(self.run_dir, "page_scores.jsonl")) + + def page_records(self) -> List[Dict[str, Any]]: + return read_jsonl(os.path.join(self.run_dir, "urls_crawled.jsonl")) + + def text_for(self, url: str) -> str: + """Page text as the crawler saved it, keyed the same way the crawler keyed it.""" + if not url: + return "" + if url in self._text_cache: + return self._text_cache[url] + path = os.path.join(self.text_dir, f"{sha1(url)}.txt") + text = "" + if os.path.exists(path): + try: + with open(path, "r", encoding="utf-8", errors="ignore") as handle: + text = handle.read() + except OSError: + text = "" + self._text_cache[url] = text + return text + + def analyses(self) -> List[Dict[str, Any]]: + if not os.path.isdir(self.analysis_dir): + return [] + rows: List[Dict[str, Any]] = [] + for name in sorted(os.listdir(self.analysis_dir)): + if not name.endswith(".analysis.json"): + continue + try: + with open(os.path.join(self.analysis_dir, name), "r", encoding="utf-8") as handle: + payload = json.load(handle) + except (OSError, ValueError): + continue + if isinstance(payload, dict): + rows.append(payload) + return rows + + +def _run_stage( + name: str, fn: Callable[[], CheckResult], output: AnalysisOutput, log: Callable[[str], None] +) -> Optional[CheckResult]: + """Execute one stage under isolation. A stage failure is reported, never propagated.""" + started = time.time() + try: + result = fn() + except Exception as exc: + message = f"{name}: stage failed: {exc}" + output.errors.append(message) + log(f"[FAIL] analysis stage {name} -> {exc}") + failed = CheckResult(check_name=name) + failed.errors.append(str(exc)) + failed.notes.append("This stage failed. Remaining stages continued.") + output.results.append(failed) + return None + + elapsed = time.time() - started + output.results.append(result) + output.findings.extend(result.findings) + output.hypotheses.extend(result.hypotheses) + output.leads.extend(result.leads) + log(f"[OK] {name}: {result.finding_count} finding(s) in {elapsed:.2f}s") + return result + + +def analyze_run( + run_dir: str, + options: AnalysisOptions, + ollama=None, + run_id: str = "", + log: Optional[Callable[[str], None]] = None, +) -> AnalysisOutput: + """Run the analysis lifecycle over a completed crawl.""" + log = log or (lambda message: None) + output = AnalysisOutput() + artifacts = RunArtifacts(run_dir) + + indicator_rows = artifacts.indicators() + page_scores = artifacts.page_scores() + page_records = artifacts.page_records() + analyses = artifacts.analyses() + analyses_by_url = {a.get("url", ""): a for a in analyses if isinstance(a, dict)} + + if not indicator_rows and not page_records: + output.errors.append("No crawl artifacts found; analysis skipped.") + log("[SKIP] analysis: no crawl artifacts to analyze") + return output + + # Entity index over the indicators the crawl already wrote. + index: EntityIndex = index_indicators(indicator_rows) + + # Extended indicator classes over saved page text. Kept out of the crawl hot path so + # the crawler's own output schema is unchanged. + page_extras: Dict[str, Dict[str, List[str]]] = {} + content_dates: Dict[str, List[str]] = {} + scanned = 0 + for record in page_records[:MAX_TEXT_PAGES]: + url = record.get("url") + if not url: + continue + text = artifacts.text_for(url) + if not text: + continue + scanned += 1 + extras = extract_extended(text) + page_extras[url] = extras + if extras["dates"]: + content_dates[url] = extras["dates"] + index.add_many("date", extras["dates"], url) + index.add_many("name", extras["name_candidates"], url) + index.add_many("address", extras["addresses"], url) + # Non-ASCII domains and handles the crawler's ASCII-only patterns cannot see. + index.add_many("domain", extras["unicode_domains"], url) + index.add_many("username", extras["unicode_handles"], url) + + # Dates the model reported are recorded too, marked by their source page. + for analysis in analyses: + url = analysis.get("url") or "" + model_dates = [d for d in (analysis.get("key_dates") or []) if isinstance(d, str)] + if url and model_dates: + content_dates.setdefault(url, []).extend(model_dates) + + output.stats["entities"] = len(index) + output.stats["pages_text_scanned"] = scanned + log(f"[OK] entity index: {len(index)} entities from {len(indicator_rows)} indicator record(s)") + + # Deterministic analysis. Cheap, offline, reproducible, always runs. + _run_stage("homoglyph analysis", lambda: patterns.check_homoglyphs(index), output, log) + _run_stage( + "sensitive infrastructure", + lambda: patterns.check_sensitive_infrastructure(index), output, log, + ) + _run_stage( + "secret exposure", + lambda: patterns.check_secret_exposure(page_extras), output, log, + ) + _run_stage( + "generated-content fingerprint", + lambda: patterns.check_generated_text(page_records, artifacts.text_for), output, log, + ) + _run_stage( + "cross-source correlation", + lambda: correlation_module.correlate(index, indicator_rows, len(page_records)), + output, log, + ) + + def temporal_stage() -> CheckResult: + events, parse_errors = temporal.build_events(page_records, content_dates) + result = temporal.analyze_timeline(events, gap_threshold_days=options.gap_days) + result.errors.extend(parse_errors[:50]) + _write_events(run_dir, events, output) + return result + + _run_stage("temporal analysis", temporal_stage, output, log) + _run_stage( + "recurring signals and outliers", + lambda: patterns.check_recurring_and_outliers(index, page_scores), output, log, + ) + + # Optional model-assisted stages. Nothing below runs unless it was asked for. + evaluation_result: Optional[CheckResult] = None + if options.evaluate: + evaluation_result = _run_stage( + "analysis quality evaluation", + lambda: evaluation_module.evaluate_run( + analyses, artifacts.text_for, model=options.model + ), + output, log, + ) + + if options.deep and options.use_ollama and ollama is not None: + _run_stage( + "deep analysis", + lambda: _deep_analysis( + ollama, options, index, output, page_scores, analyses, run_id + ), + output, log, + ) + + if options.cross_check_models and options.use_ollama and ollama is not None: + _run_stage( + "multi-model cross-check", + lambda: _cross_check(ollama, options, analyses), output, log, + ) + + # Hypotheses derive from the findings that now exist, so this runs after every stage + # that can produce one. + def hypothesis_stage() -> CheckResult: + result = CheckResult(check_name="Hypothesis Generation") + derived = hypotheses_module.from_findings(output.findings, index) + result.hypotheses.extend(derived) + result.notes.append( + f"Derived {len(derived)} hypothesis/hypotheses from deterministic findings. " + "Every entry is labelled HYPOTHESIS and is not an observed fact." + ) + return result + + _run_stage("hypothesis generation", hypothesis_stage, output, log) + + def lead_stage() -> CheckResult: + result = CheckResult(check_name="Lead Generation") + generated = pivots.generate_leads(index, per_kind=options.max_leads_per_kind) + result.leads.extend(generated) + result.notes.append( + f"Generated {len(generated)} pivot lead(s). Each is a candidate lookup path, not a hit; " + "confirmation requires opening it." + ) + return result + + _run_stage("lead generation", lead_stage, output, log) + + _write_artifacts(run_dir, output) + + if options.training_export and evaluation_result is not None: + try: + def prompt_loader(url: str) -> str: + text = artifacts.text_for(url) + if not text: + return "" + record = next((r for r in page_records if r.get("url") == url), {}) + return page_prompt(url, record.get("title", ""), text, options.prompt_profile) + + summary = training_export_module.export( + run_dir=run_dir, + evaluation_result=evaluation_result, + analyses_by_url=analyses_by_url, + prompt_loader=prompt_loader, + model=options.model, + run_id=run_id, + ) + output.artifacts["training_export"] = summary["dir"] + output.stats["training_export"] = summary["counts"] + log(f"[OK] training dataset exported: {summary['dir']}") + except Exception as exc: + output.errors.append(f"training export failed: {exc}") + log(f"[FAIL] training export -> {exc}") + elif options.training_export: + output.errors.append("training export requires --evaluate; nothing was exported.") + log("[SKIP] training export: requires --evaluate") + + output.findings = sort_findings(output.findings) + output.stats["finding_count"] = len(output.findings) + output.stats["hypothesis_count"] = len(output.hypotheses) + output.stats["lead_count"] = len(output.leads) + return output + + +def _deep_analysis( + ollama, + options: AnalysisOptions, + index: EntityIndex, + output: AnalysisOutput, + page_scores: List[Dict[str, Any]], + analyses: List[Dict[str, Any]], + run_id: str, +) -> CheckResult: + """One run-level model pass over the deterministic results. + + Optional and bounded: one call, or a small number when recursive re-analysis is + explicitly enabled. Recursive refinement is capped to keep work deterministic and bounded. + """ + import asyncio + + result = CheckResult(check_name="Deep Analysis (model-assisted)") + + top_pages = sorted(page_scores, key=lambda p: p.get("score", 0), reverse=True)[:20] + context = { + "run_id": run_id, + "pages_analyzed": len(page_scores), + "top_pages": [ + {"url": p.get("url"), "score": p.get("score"), "summary": p.get("summary")} + for p in top_pages + ], + "recurring_entities": [ + {"kind": e.kind, "value": e.value, "sources": len(e.sources)} + for e in index.multi_source(minimum=2)[:40] + ], + "deterministic_findings": [ + { + "check": f.check, "item": f.item, "reason": f.reason, + "priority": f.priority, "origin": f.origin, + } + for f in sort_findings(output.findings)[:60] + ], + } + + sources = [p.get("url", "") for p in top_pages if p.get("url")] + passes = 1 + max(0, min(3, options.experimental_recursive)) + payload: Optional[Dict[str, Any]] = None + + for pass_number in range(passes): + prompt = deep_analysis_prompt(context) + try: + payload = asyncio.run( + ollama.async_generate_json(options.model, prompt, timeout_s=180.0) + ) + except RuntimeError as exc: + result.errors.append(f"deep analysis pass {pass_number + 1} could not run: {exc}") + break + if not payload: + result.errors.append( + f"deep analysis pass {pass_number + 1} returned no parseable response." + ) + break + + result.rows.append({ + "pass": pass_number + 1, + "model": options.model, + "assessment": payload.get("assessment", ""), + "cross_source_observations": payload.get("cross_source_observations", []), + "recommended_follow_up": payload.get("recommended_follow_up", []), + "gaps": payload.get("gaps", []), + }) + result.hypotheses.extend( + hypotheses_module.from_model(payload, options.model, sources) + ) + + if pass_number + 1 >= passes: + break + # Experimental recursion feeds the prior pass back as context, bounded above. + context = { + "run_id": run_id, + "previous_pass": payload, + "deterministic_findings": context["deterministic_findings"], + "instruction": "Refine the previous assessment. Remove anything unsupported.", + } + + if payload is None and not result.errors: + result.errors.append("Deep analysis produced no output.") + + result.stats = {"passes_run": len(result.rows), "model": options.model} + result.notes.append( + f"Deep analysis ran {len(result.rows)} model pass(es) with {options.model or 'the configured model'}. " + "All output is model-generated interpretation, labelled as such, and is not observed fact." + ) + return result + + +def _cross_check(ollama, options: AnalysisOptions, analyses: List[Dict[str, Any]]) -> CheckResult: + import asyncio + + claims = multimodel.claims_from_analyses(analyses) + if not claims: + result = CheckResult(check_name="Multi-Model Cross-Check") + result.notes.append("No model claims available to cross-check.") + return result + + models = list(dict.fromkeys([options.model, *options.cross_check_models])) + models = [m for m in models if m] + try: + checks = asyncio.run(multimodel.cross_check_claims(ollama, models, claims)) + except RuntimeError as exc: + result = CheckResult(check_name="Multi-Model Cross-Check") + result.errors.append(f"cross-check could not run: {exc}") + return result + return multimodel.summarize_cross_checks(checks) + + +def _write_events(run_dir: str, events: List[temporal.Event], output: AnalysisOutput) -> None: + path = os.path.join(run_dir, "timeline.jsonl") + if os.path.exists(path): + os.remove(path) + for event in events: + append_jsonl(path, event.to_dict()) + output.artifacts["timeline"] = path + + +def _write_artifacts(run_dir: str, output: AnalysisOutput) -> None: + """Write the analysis artifacts. Existing crawl artifacts are never rewritten.""" + for name, rows in ( + ("findings.jsonl", [f.to_dict() for f in sort_findings(output.findings)]), + ("hypotheses.jsonl", [h.to_dict() for h in output.hypotheses]), + ("leads.jsonl", [l.to_dict() for l in output.leads]), + ): + path = os.path.join(run_dir, name) + if os.path.exists(path): + os.remove(path) + for row in rows: + append_jsonl(path, row) + output.artifacts[name.split(".")[0]] = path + + correlations = next( + (r for r in output.results if r.check_name == "Cross-Source Correlation"), None + ) + if correlations is not None: + path = os.path.join(run_dir, "correlations.jsonl") + if os.path.exists(path): + os.remove(path) + for row in correlations.rows: + append_jsonl(path, row) + output.artifacts["correlations"] = path + + summary_path = os.path.join(run_dir, "analysis_summary.json") + write_json(summary_path, { + "stages": [ + { + "check_name": r.check_name, + "findings": r.finding_count, + "notes": r.notes, + "errors": r.errors, + "stats": r.stats, + } + for r in output.results + ], + "stats": output.stats, + "errors": output.errors, + }) + output.artifacts["analysis_summary"] = summary_path diff --git a/src/osintai/pivots.py b/src/osintai/pivots.py new file mode 100644 index 0000000..99ccb40 --- /dev/null +++ b/src/osintai/pivots.py @@ -0,0 +1,218 @@ +"""Lead generation: turn an observed identifier into the next place to look. + +This module emits real lookup URLs without inventing results. Whether a lookup resolves to +the subject is something the investigator establishes by opening it. A generated pivot is a +candidate, never a hit, and this module itself makes no network requests. +""" + +from __future__ import annotations + +from typing import Any, Dict, List +from urllib.parse import quote_plus + +from .entities import ( + CRYPTO_BTC, + CRYPTO_ETH, + DOMAIN, + EMAIL, + Entity, + EntityIndex, + IP, + NAME, + PHONE, + USERNAME, + normalize_phone, + phone_variants, +) +from .provenance import Lead + +# How many entities of each kind get expanded. A crawl can produce thousands of identifiers +# and a lead list nobody can work through is not a lead list. +DEFAULT_PER_KIND = 10 + + +def _q(value: str) -> str: + return quote_plus(value) + + +def whois_url(domain: str) -> str: + return f"https://www.whois.com/whois/{_q(domain)}" + + +def crtsh_url(domain: str) -> str: + return f"https://crt.sh/?q={_q(domain)}" + + +def shodan_domain_url(domain: str) -> str: + return f"https://www.shodan.io/search?query={_q(domain)}" + + +def shodan_host_url(ip: str) -> str: + return f"https://www.shodan.io/host/{_q(ip)}" + + +def _search_engines(query: str) -> List[Dict[str, str]]: + quoted = f'"{query}"' + return [ + {"label": "Google", "target": f"https://www.google.com/search?q={_q(quoted)}"}, + {"label": "Bing", "target": f"https://www.bing.com/search?q={_q(quoted)}"}, + {"label": "DuckDuckGo", "target": f"https://duckduckgo.com/?q={_q(quoted)}"}, + ] + + +def pivots_for_username(handle: str) -> List[Dict[str, str]]: + u = handle.lstrip("@") + q = _q(u) + return [ + {"label": "GitHub profile", "target": f"https://github.com/{q}", + "rationale": "A 404 establishes the account does not exist."}, + {"label": "Reddit profile", "target": f"https://www.reddit.com/user/{q}", + "rationale": "A 404 establishes the account does not exist."}, + {"label": "X / Twitter", "target": f"https://x.com/{q}"}, + {"label": "Instagram", "target": f"https://www.instagram.com/{q}/"}, + {"label": "Telegram", "target": f"https://t.me/{q}"}, + {"label": "Keybase", "target": f"https://keybase.io/{q}"}, + {"label": "WhatsMyName enumeration", "target": f"https://whatsmyname.app/?q={q}", + "rationale": "Checks the handle across several hundred sites in one pass."}, + *_search_engines(u), + ] + + +def pivots_for_email(email: str) -> List[Dict[str, str]]: + q = _q(email) + local = email.split("@")[0] if "@" in email else "" + domain = email.split("@")[-1] if "@" in email else "" + pivots = [ + {"label": "Have I Been Pwned", "target": f"https://haveibeenpwned.com/account/{q}", + "rationale": "Breach exposure history for the address."}, + {"label": "IntelligenceX", "target": f"https://intelx.io/?s={q}"}, + {"label": "Gravatar", "target": f"https://gravatar.com/{q}", + "rationale": "A Gravatar often carries a real name and photograph."}, + *_search_engines(email), + ] + if local and len(local) >= 4: + pivots.append({ + "label": f"Pivot local part {local!r} as a handle", + "target": f"https://whatsmyname.app/?q={_q(local)}", + "rationale": "Local parts are frequently reused as usernames elsewhere.", + }) + if domain: + pivots.append({"label": "Sending domain WHOIS", "target": whois_url(domain)}) + return pivots + + +def pivots_for_phone(phone: str) -> List[Dict[str, str]]: + digits = normalize_phone(phone)["digits"] + if not digits: + return [] + variants = phone_variants(phone) + or_query = " OR ".join(f'"{v}"' for v in variants[:6]) + return [ + {"label": "TrueCaller", "target": f"https://www.truecaller.com/search/us/{_q(digits)}"}, + {"label": "TruePeopleSearch", "target": + f"https://www.truepeoplesearch.com/resultphone?phoneno={_q(digits)}"}, + {"label": "ThatsThem", "target": f"https://thatsthem.com/phone/{_q(digits)}"}, + {"label": "WhatsApp presence", "target": f"https://wa.me/{_q(digits)}", + "rationale": "Establishes whether the number is registered to the service."}, + {"label": "Search all written forms", + "target": f"https://www.google.com/search?q={_q(or_query)}", + "rationale": "Covers the formats the number is likely to be written in."}, + ] + + +def pivots_for_name(name: str) -> List[Dict[str, str]]: + q = _q(name) + parts = name.split() + slug = _q("-".join(parts)) if len(parts) >= 2 else q + return [ + {"label": "TruePeopleSearch", "target": f"https://www.truepeoplesearch.com/results?name={q}"}, + {"label": "FastPeopleSearch", "target": f"https://www.fastpeoplesearch.com/name/{slug}"}, + {"label": "ThatsThem", "target": f"https://thatsthem.com/name/{slug}"}, + {"label": "LinkedIn people search", + "target": f"https://www.linkedin.com/search/results/people/?keywords={q}"}, + *_search_engines(name), + ] + + +def pivots_for_domain(domain: str) -> List[Dict[str, str]]: + return [ + {"label": "WHOIS registration", "target": whois_url(domain), + "rationale": "Registrant, registrar, and creation date."}, + {"label": "Certificate transparency", "target": crtsh_url(domain), + "rationale": "Historic and sibling hostnames issued under the domain."}, + {"label": "Shodan", "target": shodan_domain_url(domain), + "rationale": "Hosting infrastructure and exposed services."}, + {"label": "Wayback history", "target": f"https://web.archive.org/web/*/{domain}", + "rationale": "What the site said before it said what it says now."}, + ] + + +def pivots_for_ip(ip: str) -> List[Dict[str, str]]: + return [ + {"label": "Shodan host", "target": shodan_host_url(ip)}, + {"label": "Reverse DNS and hosting", "target": f"https://viewdns.info/reverseip/?host={_q(ip)}"}, + ] + + +def pivots_for_crypto(address: str, chain: str) -> List[Dict[str, str]]: + if chain == CRYPTO_ETH: + return [{"label": "Etherscan", "target": f"https://etherscan.io/address/{_q(address)}", + "rationale": "Transaction history and counterparties."}] + return [{"label": "Blockchain explorer", + "target": f"https://www.blockchain.com/explorer/addresses/btc/{_q(address)}", + "rationale": "Transaction history and counterparties."}] + + +# False-positive risk notes, because a pivot without one invites over-reading a hit. +FALSE_POSITIVE_RISK = { + USERNAME: "High. Handles collide constantly; identical handles on two sites are often different people.", + EMAIL: "Low for the address itself, high for anything pivoted from its local part.", + PHONE: "Medium. Numbers are reassigned, and business numbers are shared by many people.", + NAME: "Very high. Name matching alone establishes nothing without a second corroborating identifier.", + DOMAIN: "Low for registration data, medium for shared-hosting inferences.", + IP: "High. Shared hosting and CDNs put unrelated parties behind one address.", + CRYPTO_BTC: "Low for the address, high for attributing an address to a person.", + CRYPTO_ETH: "Low for the address, high for attributing an address to a person.", +} + +_BUILDERS = { + USERNAME: lambda v: pivots_for_username(v), + EMAIL: lambda v: pivots_for_email(v), + PHONE: lambda v: pivots_for_phone(v), + NAME: lambda v: pivots_for_name(v), + DOMAIN: lambda v: pivots_for_domain(v), + IP: lambda v: pivots_for_ip(v), + CRYPTO_BTC: lambda v: pivots_for_crypto(v, CRYPTO_BTC), + CRYPTO_ETH: lambda v: pivots_for_crypto(v, CRYPTO_ETH), +} + + +def pivots_for(kind: str, value: str) -> List[Dict[str, str]]: + builder = _BUILDERS.get(kind) + return builder(value) if builder else [] + + +def generate_leads(index: EntityIndex, per_kind: int = DEFAULT_PER_KIND) -> List[Lead]: + """Expand the best-corroborated entities of each kind into actionable pivots. + + Entities seen on more pages come first, on the reasoning that an identifier the crawl + kept running into is the one most worth chasing. + """ + leads: List[Lead] = [] + for kind in _BUILDERS: + entities = sorted( + index.of_kind(kind), + key=lambda e: (-len(e.sources), -e.count, e.canonical_value), + )[:per_kind] + for entity in entities: + for pivot in pivots_for(kind, entity.value): + leads.append(Lead( + label=pivot["label"], + target=pivot["target"], + rationale=pivot.get("rationale", ""), + seed=entity.value, + seed_type=kind, + sources=list(entity.sources)[:10], + false_positive_risk=FALSE_POSITIVE_RISK.get(kind, ""), + )) + return leads diff --git a/src/osintai/prompts.py b/src/osintai/prompts.py new file mode 100644 index 0000000..c432c73 --- /dev/null +++ b/src/osintai/prompts.py @@ -0,0 +1,170 @@ +"""Prompt registry for OSINTai. + +The `standard` page prompt is the exact text the crawler has always sent, moved here +unchanged so that `osint-tuned-v3` sees byte-identical input and its behavior does not +shift. A regression test asserts that equality; do not edit the standard profile. + +Additional profiles are opt-in lenses over the same page. The `threat` profile asks a +different question of the same content without silently changing the default. Run-level +prompts keep confidence, evidence, and proposed pivots distinct. +""" + +from __future__ import annotations + +from typing import Any, Callable, Dict, List + +STANDARD = "standard" +THREAT = "threat" + +PAGE_SNIPPET_CHARS = 12000 + +# Stated at the top of every run-level prompt. Kept out of the standard page prompt so that +# prompt stays byte-identical to the baseline. +ANALYST_PREAMBLE = """You are a professional OSINT analyst. + +Your output must: +- Separate confirmed facts from unverified leads +- Use precise confidence language: documented, indicated, appears, possible, unverified, requires confirmation +- Never overstate the strength of a lead +- Reference the source URL for every finding +- Flag false-positive risk on identifier-based pivots + +Do not speculate. Do not present leads as facts. Output valid JSON only.""" + + +def standard_page_prompt(url: str, title: str, text: str) -> str: + """The baseline per-page extraction prompt. Text is frozen; do not modify.""" + snippet = (text or "")[:PAGE_SNIPPET_CHARS] + return f""" +You are an OSINT analyst. Produce a structured intelligence extraction from this webpage. + +Rules: +- No filler. No moralizing. +- Only use evidence from the content. +- Output valid JSON only. + +Return schema: +{{ + "url": "...", + "title": "...", + "summary": "2-4 sentences", + "key_entities": ["..."], + "key_locations": ["..."], + "key_dates": ["..."], + "keywords": ["..."], + "risk_flags": ["..."], + "actionable_leads": ["..."] +}} + +URL: {url} +TITLE: {title} + +CONTENT: +{snippet} +""".strip() + + +def threat_page_prompt(url: str, title: str, text: str) -> str: + """Cyber-threat lens. Same JSON schema so every downstream consumer keeps working.""" + snippet = (text or "")[:PAGE_SNIPPET_CHARS] + return f""" +You are a cyber threat analyst. Assess this webpage for behavioral and cyber threat indicators. + +Rules: +- No filler. No moralizing. +- Only use evidence from the content. +- Output valid JSON only. + +Return schema: +{{ + "url": "...", + "title": "...", + "summary": "2-4 sentences covering threats and suspicious behavior", + "key_entities": ["actors, organizations, infrastructure named in the content"], + "key_locations": ["..."], + "key_dates": ["..."], + "keywords": ["..."], + "risk_flags": ["indicators of compromise, suspicious behavior, exposure"], + "actionable_leads": ["recommended further investigation"] +}} + +URL: {url} +TITLE: {title} + +CONTENT: +{snippet} +""".strip() + + +PAGE_PROFILES: Dict[str, Callable[[str, str, str], str]] = { + STANDARD: standard_page_prompt, + THREAT: threat_page_prompt, +} + + +def page_prompt(url: str, title: str, text: str, profile: str = STANDARD) -> str: + builder = PAGE_PROFILES.get(profile, standard_page_prompt) + return builder(url, title, text) + + +def deep_analysis_prompt(context: Dict[str, Any]) -> str: + """Run-level analysis over the whole crawl. + + The context payload assembles several independent sources into one structured block and + asks the model to reason across them — the cross-source fusion pattern from AvantGarde, + applied to crawled material rather than third-party APIs. + """ + import json + + return f"""{ANALYST_PREAMBLE} + +You are reviewing an entire OSINT collection run. Deterministic analysis has already been +performed and is given to you below. Do not repeat it. Identify what it does not already say. + +Return schema: +{{ + "assessment": "3-6 sentences on what this collection appears to show", + "cross_source_observations": ["patterns visible only across multiple sources"], + "hypotheses": [ + {{ + "statement": "...", + "supporting": ["..."], + "contradicting": ["..."], + "would_confirm": "...", + "would_refute": "...", + "follow_up": "...", + "confidence": 0.0 + }} + ], + "recommended_follow_up": ["..."], + "gaps": ["what this collection does not cover"] +}} + +RUN CONTEXT: +{json.dumps(context, ensure_ascii=False, indent=2, default=str)[:24000]} +""".strip() + + +def cross_check_prompt(claim: str, context: str) -> str: + """Ask a second model to judge one claim independently. + + Deliberately narrow: a single claim and its context, so two models' answers are + comparable rather than two different essays. + """ + return f"""{ANALYST_PREAMBLE} + +Assess the single claim below against the supplied context only. + +Return schema: +{{ + "verdict": "supported" | "unsupported" | "contradicted" | "insufficient_evidence", + "reason": "1-2 sentences", + "confidence": 0.0 +}} + +CLAIM: +{claim} + +CONTEXT: +{context[:8000]} +""".strip() diff --git a/src/osintai/provenance.py b/src/osintai/provenance.py new file mode 100644 index 0000000..5ab4f08 --- /dev/null +++ b/src/osintai/provenance.py @@ -0,0 +1,261 @@ +"""Provenance model for OSINTai analysis. + +Every analytical statement OSINTai produces carries where it came from. A regex hit on a +crawled page, a calculation over those hits, something a local model said, and a proposed +explanation are four different kinds of claim, and the operator has to be able to tell them +apart in the report without reading the code that produced them. + +Findings carry explicit origin, source, and confidence fields so observed data remains +separate from derived and model-assisted analysis. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + +# Origin of a claim. Ordered weakest-commitment last. +OBSERVED = "OBSERVED" # present in fetched source material +DERIVED = "DERIVED" # computed deterministically from observed material +MODEL = "MODEL" # produced by a language model +HYPOTHESIS = "HYPOTHESIS" # a proposed explanation, not a finding + +ORIGINS = (OBSERVED, DERIVED, MODEL, HYPOTHESIS) + +# Investigator-facing priority. +HIGH = "HIGH" +MEDIUM = "MEDIUM" +LOW = "LOW" + +PRIORITY_ORDER = {HIGH: 0, MEDIUM: 1, LOW: 2} + +# Confidence kinds are deliberately not interchangeable. A model reporting that it is +# certain is not the same class of evidence as three independent pages agreeing, and +# collapsing them into one number is how model output turns into apparent fact. +SOURCE_SUPPORT = "source_support" # how many independent sources carry it +DETERMINISTIC = "deterministic" # reproducible rule fired +MODEL_SELF = "model_self" # the model's own stated confidence +CROSS_MODEL = "cross_model" # independent models agreed +HUMAN_REVIEW = "human_review" # an operator confirmed it + +CONFIDENCE_KINDS = (SOURCE_SUPPORT, DETERMINISTIC, MODEL_SELF, CROSS_MODEL, HUMAN_REVIEW) + + +def _clamp(value: float) -> float: + try: + value = float(value) + except (TypeError, ValueError): + return 0.0 + return max(0.0, min(1.0, value)) + + +@dataclass +class Confidence: + """A single confidence signal. Never merged across kinds.""" + + kind: str + value: float + basis: str = "" + + def __post_init__(self) -> None: + if self.kind not in CONFIDENCE_KINDS: + raise ValueError(f"unknown confidence kind: {self.kind!r}") + self.value = _clamp(self.value) + + def to_dict(self) -> Dict[str, Any]: + return {"kind": self.kind, "value": round(self.value, 4), "basis": self.basis} + + +@dataclass +class Finding: + """One analytical statement with its provenance and its recommended next step. + + `next_step` is required in spirit: a finding an operator cannot act on is noise. + """ + + check: str + item: str + reason: str + next_step: str + origin: str = DERIVED + priority: str = MEDIUM + sources: List[str] = field(default_factory=list) + evidence: Dict[str, Any] = field(default_factory=dict) + method: str = "" + model: str = "" + confidence: List[Confidence] = field(default_factory=list) + ts: float = field(default_factory=time.time) + + def __post_init__(self) -> None: + if self.origin not in ORIGINS: + raise ValueError(f"unknown origin: {self.origin!r}") + if self.priority not in PRIORITY_ORDER: + raise ValueError(f"unknown priority: {self.priority!r}") + + @property + def is_model_derived(self) -> bool: + return self.origin in (MODEL, HYPOTHESIS) + + def to_dict(self) -> Dict[str, Any]: + return { + "origin": self.origin, + "priority": self.priority, + "check": self.check, + "item": self.item, + "reason": self.reason, + "next_step": self.next_step, + "sources": self.sources, + "evidence": self.evidence, + "method": self.method, + "model": self.model, + "confidence": [c.to_dict() for c in self.confidence], + "ts": self.ts, + } + + +@dataclass +class Hypothesis: + """A proposed explanation. Always labelled, never scored as an observation. + + Carries what would settle it, which is the part that makes a hypothesis useful to an + investigator rather than just suggestive. + """ + + statement: str + supporting: List[str] = field(default_factory=list) + contradicting: List[str] = field(default_factory=list) + sources: List[str] = field(default_factory=list) + would_confirm: str = "" + would_refute: str = "" + follow_up: str = "" + method: str = "" + model: str = "" + confidence: List[Confidence] = field(default_factory=list) + ts: float = field(default_factory=time.time) + + # Fixed. A hypothesis cannot be relabelled into an observation. + origin: str = HYPOTHESIS + + def to_dict(self) -> Dict[str, Any]: + return { + "origin": HYPOTHESIS, + "label": HYPOTHESIS, + "statement": self.statement, + "supporting": self.supporting, + "contradicting": self.contradicting, + "sources": self.sources, + "would_confirm": self.would_confirm, + "would_refute": self.would_refute, + "follow_up": self.follow_up, + "method": self.method, + "model": self.model, + "confidence": [c.to_dict() for c in self.confidence], + "ts": self.ts, + } + + +@dataclass +class Lead: + """An actionable next action with a resolvable target.""" + + label: str + target: str + rationale: str = "" + seed: str = "" + seed_type: str = "" + sources: List[str] = field(default_factory=list) + false_positive_risk: str = "" + origin: str = DERIVED + ts: float = field(default_factory=time.time) + + def to_dict(self) -> Dict[str, Any]: + return { + "origin": self.origin, + "label": self.label, + "target": self.target, + "rationale": self.rationale, + "seed": self.seed, + "seed_type": self.seed_type, + "sources": self.sources, + "false_positive_risk": self.false_positive_risk, + "ts": self.ts, + } + + +@dataclass +class CheckResult: + """Output of one analysis stage. + + Stages collect their problems instead of raising them. A missing input is a note, a bad + record is an error, and either way the remaining stages still run and still report. + """ + + check_name: str + rows: List[Dict[str, Any]] = field(default_factory=list) + findings: List[Finding] = field(default_factory=list) + hypotheses: List[Hypothesis] = field(default_factory=list) + leads: List[Lead] = field(default_factory=list) + notes: List[str] = field(default_factory=list) + errors: List[str] = field(default_factory=list) + stats: Dict[str, Any] = field(default_factory=dict) + + @property + def finding_count(self) -> int: + return len(self.findings) + + def to_dict(self) -> Dict[str, Any]: + return { + "check_name": self.check_name, + "rows": self.rows, + "findings": [f.to_dict() for f in self.findings], + "hypotheses": [h.to_dict() for h in self.hypotheses], + "leads": [l.to_dict() for l in self.leads], + "notes": self.notes, + "errors": self.errors, + "stats": self.stats, + } + + +def sort_findings(findings: List[Finding]) -> List[Finding]: + """Highest priority first, then stable by check and item.""" + return sorted( + findings, + key=lambda f: (PRIORITY_ORDER.get(f.priority, 99), f.check, str(f.item)), + ) + + +def source_support(sources: List[str]) -> Confidence: + """Confidence from breadth of independent corroboration. + + One page saying something is one page saying something. The curve is deliberately + shallow: four independent sources reach 0.8, and nothing reaches 1.0, because crawled + corroboration is not confirmation. + """ + n = len({s for s in sources if s}) + if n <= 0: + return Confidence(SOURCE_SUPPORT, 0.0, "no source reference") + value = min(0.9, 0.35 + 0.15 * (n - 1)) + return Confidence(SOURCE_SUPPORT, value, f"{n} independent source(s)") + + +def deterministic_confidence(value: float, basis: str) -> Confidence: + return Confidence(DETERMINISTIC, value, basis) + + +def model_self_confidence(value: float, model: str) -> Confidence: + """The model's own stated confidence. Recorded, labelled, and never treated as fact.""" + return Confidence(MODEL_SELF, value, f"self-reported by {model or 'model'}") + + +def cross_model_confidence(agreeing: int, total: int) -> Confidence: + if total <= 1: + return Confidence(CROSS_MODEL, 0.0, "single model, no cross-check") + return Confidence( + CROSS_MODEL, agreeing / total, f"{agreeing}/{total} models agreed" + ) + + +def strongest(confidences: List[Confidence], kind: Optional[str] = None) -> Optional[Confidence]: + pool = [c for c in confidences if kind is None or c.kind == kind] + return max(pool, key=lambda c: c.value) if pool else None diff --git a/src/osintai/proxy_pool.py b/src/osintai/proxy_pool.py index 4b1b047..9729d8f 100644 --- a/src/osintai/proxy_pool.py +++ b/src/osintai/proxy_pool.py @@ -1,4 +1,4 @@ -import random +import secrets import time from typing import List, Dict, Optional from dataclasses import dataclass @@ -43,7 +43,7 @@ def pick(self) -> Optional[str]: # Weighted by score, but also random entries = sorted(self.entries, key=lambda e: e.score, reverse=True) top = entries[: max(3, len(entries) // 5)] - chosen = random.choice(top) + chosen = secrets.choice(top) chosen.last_used = time.time() return chosen.url diff --git a/src/osintai/report.py b/src/osintai/report.py index 2918c3e..3c77759 100644 --- a/src/osintai/report.py +++ b/src/osintai/report.py @@ -1,6 +1,23 @@ import os +from typing import Any, Dict, List, Optional + +from .provenance import ( + DERIVED, + HYPOTHESIS, + MODEL, + OBSERVED, + PRIORITY_ORDER, + Finding, + Hypothesis, + Lead, + sort_findings, +) from .storage import write_json +RULE = "=" * 60 +THIN = "-" * 60 + + def write_report(run_dir: str, ranked_pages: list): out_txt = os.path.join(run_dir, "report.txt") lines = [] @@ -19,4 +36,242 @@ def write_report(run_dir: str, ranked_pages: list): with open(out_txt, "w", encoding="utf-8") as f: f.write("\n".join(lines)) write_json(os.path.join(run_dir, "ranked_pages.json"), ranked_pages) - return out_txt \ No newline at end of file + return out_txt + + +# --------------------------------------------------------------------------- +# Analysis report +# +# report.txt above is unchanged and stays the ranked-pages report every existing +# consumer expects. The analysis report is a separate file covering what the analysis +# stage produced. +# +# An operator must be able to distinguish what a page said from what code computed and from +# what a model proposed without reading the implementation. +# --------------------------------------------------------------------------- + +ORIGIN_SECTIONS = ( + (OBSERVED, "OBSERVED — PRESENT IN SOURCE MATERIAL"), + (DERIVED, "DERIVED — COMPUTED FROM SOURCE MATERIAL"), + (MODEL, "MODEL-ASSISTED — LANGUAGE MODEL INTERPRETATION"), +) + + +def _wrap(text: str, width: int = 96, indent: str = " ") -> List[str]: + words = str(text or "").split() + if not words: + return [] + lines, current = [], "" + for word in words: + if current and len(current) + 1 + len(word) > width: + lines.append(indent + current) + current = word + else: + current = f"{current} {word}".strip() + if current: + lines.append(indent + current) + return lines + + +def _finding_block(finding: Finding, number: int) -> List[str]: + lines = [f" {number:02d}. [{finding.priority}] {finding.check}: {finding.item}"] + lines.extend(_wrap(f"Why: {finding.reason}")) + lines.extend(_wrap(f"Next step: {finding.next_step}")) + if finding.method: + lines.append(f" Method: {finding.method}" + (f" (model: {finding.model})" if finding.model else "")) + for confidence in finding.confidence: + lines.append( + f" Confidence [{confidence.kind}]: {confidence.value:.2f} — {confidence.basis}" + ) + if finding.sources: + lines.append(f" Sources ({len(finding.sources)}):") + for source in finding.sources[:5]: + lines.append(f" - {source}") + if len(finding.sources) > 5: + lines.append(f" ... and {len(finding.sources) - 5} more") + lines.append("") + return lines + + +def _hypothesis_block(hypothesis: Hypothesis, number: int) -> List[str]: + lines = [f" {number:02d}. [HYPOTHESIS] {hypothesis.statement}"] + for support in hypothesis.supporting[:4]: + lines.extend(_wrap(f"Supporting: {support}")) + for against in hypothesis.contradicting[:4]: + lines.extend(_wrap(f"Contradicting: {against}")) + if hypothesis.would_confirm: + lines.extend(_wrap(f"Would confirm: {hypothesis.would_confirm}")) + if hypothesis.would_refute: + lines.extend(_wrap(f"Would refute: {hypothesis.would_refute}")) + if hypothesis.follow_up: + lines.extend(_wrap(f"Follow up: {hypothesis.follow_up}")) + if hypothesis.model: + lines.append(f" Origin: model-generated ({hypothesis.model})") + elif hypothesis.method: + lines.append(f" Origin: derived from findings ({hypothesis.method})") + for confidence in hypothesis.confidence: + lines.append( + f" Confidence [{confidence.kind}]: {confidence.value:.2f} — {confidence.basis}" + ) + if hypothesis.sources: + lines.append(f" Sources ({len(hypothesis.sources)}):") + for source in hypothesis.sources[:4]: + lines.append(f" - {source}") + lines.append("") + return lines + + +def _section(title: str) -> List[str]: + return ["", RULE, title, RULE, ""] + + +def write_analysis_report( + run_dir: str, + output, + run_id: str = "", + scope: Optional[Dict[str, Any]] = None, +) -> str: + """Write analysis_report.txt covering findings, correlations, timeline, hypotheses and leads.""" + lines: List[str] = [] + findings = sort_findings(list(output.findings)) + + lines.append("OSINTai Analysis Report") + lines.append(RULE) + if run_id: + lines.append(f"RUN ID: {run_id}") + lines.append(f"RUN DIR: {run_dir}") + + # SCOPE + lines.extend(_section("SCOPE")) + for key, value in (scope or {}).items(): + lines.append(f" {key}: {value}") + stats = output.stats or {} + lines.append(f" entities indexed: {stats.get('entities', 0)}") + lines.append(f" page texts scanned: {stats.get('pages_text_scanned', 0)}") + lines.append(f" findings: {len(findings)}") + lines.append(f" hypotheses: {len(output.hypotheses)}") + lines.append(f" leads: {len(output.leads)}") + + # HOW TO READ + lines.extend(_section("HOW TO READ THIS REPORT")) + lines.append(" OBSERVED present in the fetched source material.") + lines.append(" DERIVED computed deterministically from observed material. Reproducible.") + lines.append(" MODEL a language model's interpretation. Not a measurement.") + lines.append(" HYPOTHESIS a proposed explanation. Not a finding.") + lines.append("") + lines.append(" Confidence signals are reported by kind and are not interchangeable.") + lines.append(" A model's self-reported confidence is not evidence of reliability.") + + # STAGES + lines.extend(_section("ANALYSIS STAGES")) + for result in output.results: + lines.append(f" {result.check_name}: {result.finding_count} finding(s)") + for note in result.notes: + lines.extend(_wrap(note, indent=" ")) + for error in result.errors[:5]: + lines.append(f" Error: {error}") + if len(result.errors) > 5: + lines.append(f" ... and {len(result.errors) - 5} more error(s)") + lines.append("") + + # FINDINGS BY ORIGIN + for origin, title in ORIGIN_SECTIONS: + subset = [f for f in findings if f.origin == origin] + lines.extend(_section(title)) + if not subset: + lines.append(" No findings in this category.") + continue + for number, finding in enumerate(subset, start=1): + lines.extend(_finding_block(finding, number)) + + # CORRELATIONS + correlations = next( + (r for r in output.results if r.check_name == "Cross-Source Correlation"), None + ) + lines.extend(_section("CROSS-SOURCE CORRELATIONS")) + if correlations is None or not correlations.rows: + lines.append(" No correlation candidates identified.") + else: + lines.append(" All entries are CANDIDATE links. No entities were merged.") + lines.append("") + for number, row in enumerate(correlations.rows[:40], start=1): + left = row.get("left", {}) + right = row.get("right", {}) + lines.append( + f" {number:02d}. [{row.get('score', 0):.2f}] {left.get('value')} <-> {right.get('value')}" + ) + lines.append(f" relation: {row.get('relation')} status: {row.get('status')}") + lines.extend(_wrap(row.get("rationale", ""))) + lines.append(f" evidence pages: {row.get('evidence_count', 0)}") + lines.append("") + if len(correlations.rows) > 40: + lines.append(f" ... and {len(correlations.rows) - 40} more in correlations.jsonl") + + # TIMELINE + temporal_result = next( + (r for r in output.results if r.check_name == "Temporal Analysis"), None + ) + lines.extend(_section("TIMELINE")) + if temporal_result is None or not temporal_result.rows: + lines.append(" No dated events recovered.") + else: + stats = temporal_result.stats or {} + lines.append(f" Events: {stats.get('event_count', 0)}") + lines.append(f" Content dates: {stats.get('content_date_count', 0)}") + lines.append(f" Window: {stats.get('window_start', '')} to {stats.get('window_end', '')}") + lines.append(f" Span: {stats.get('window_days', 0)} day(s)") + lines.append("") + lines.append(" Full ordered event stream: timeline.jsonl") + + # HYPOTHESES + lines.extend(_section("HYPOTHESES — NOT FINDINGS, NOT FACTS")) + if not output.hypotheses: + lines.append(" No hypotheses generated.") + else: + for number, hypothesis in enumerate(output.hypotheses, start=1): + lines.extend(_hypothesis_block(hypothesis, number)) + + # LEADS + lines.extend(_section("LEADS — CANDIDATE LOOKUP PATHS")) + if not output.leads: + lines.append(" No leads generated.") + else: + lines.append(" A lead is a place to look, not a result. Opening it is how it is confirmed.") + lines.append("") + by_seed: Dict[str, List[Lead]] = {} + for lead in output.leads: + by_seed.setdefault(f"{lead.seed_type}: {lead.seed}", []).append(lead) + for seed, seed_leads in list(by_seed.items())[:40]: + lines.append(f" {seed}") + risk = seed_leads[0].false_positive_risk + if risk: + lines.extend(_wrap(f"False-positive risk: {risk}")) + for lead in seed_leads[:8]: + lines.append(f" - {lead.label}: {lead.target}") + if lead.rationale: + lines.extend(_wrap(lead.rationale, indent=" ")) + lines.append("") + if len(by_seed) > 40: + lines.append(f" ... and {len(by_seed) - 40} more seeds in leads.jsonl") + + # RECOMMENDED NEXT STEPS + lines.extend(_section("RECOMMENDED NEXT STEPS")) + if not findings: + lines.append(" No findings. Preserve the run artifacts and widen scope or depth to collect more.") + else: + for number, finding in enumerate(findings[:30], start=1): + lines.append(f" {number:02d}. [{finding.priority}] {finding.check} — {finding.item}") + lines.extend(_wrap(finding.next_step, indent=" ")) + if len(findings) > 30: + lines.append(f" ... and {len(findings) - 30} more in findings.jsonl") + + if output.errors: + lines.extend(_section("ANALYSIS ERRORS")) + for error in output.errors: + lines.append(f" - {error}") + + lines.append("") + out_txt = os.path.join(run_dir, "analysis_report.txt") + with open(out_txt, "w", encoding="utf-8") as handle: + handle.write("\n".join(lines)) + return out_txt diff --git a/src/osintai/storage.py b/src/osintai/storage.py index aed17bf..13be1fa 100644 --- a/src/osintai/storage.py +++ b/src/osintai/storage.py @@ -1,6 +1,7 @@ import os import json import hashlib +import tempfile from datetime import datetime def safe_mkdir(path: str): @@ -19,8 +20,8 @@ def load_lines(path: str) -> list[str]: return [line.strip() for line in f if line.strip()] def sha1(text: str) -> str: - """Compute SHA1 hash of text.""" - return hashlib.sha1(text.encode("utf-8")).hexdigest() + """Compute a stable legacy artifact ID; never used for security.""" + return hashlib.sha1(text.encode("utf-8"), usedforsecurity=False).hexdigest() def append_jsonl(path: str, data: dict): """Append a JSON object as a line to a file.""" @@ -31,10 +32,31 @@ def read_json(path: str, default=None) -> dict: """Read JSON from file.""" if not os.path.exists(path): return default if default is not None else {} - with open(path, "r", encoding="utf-8") as f: - return json.load(f) + try: + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + except (OSError, ValueError, TypeError): + return default if default is not None else {} def write_json(path: str, data: dict): - """Write JSON to file.""" - with open(path, "w", encoding="utf-8") as f: - json.dump(data, f, indent=2, ensure_ascii=False) \ No newline at end of file + """Write JSON atomically so an interrupted checkpoint cannot corrupt prior state.""" + parent = os.path.dirname(os.path.abspath(path)) + safe_mkdir(parent) + temp_path = "" + try: + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + dir=parent, + prefix=f".{os.path.basename(path)}.", + suffix=".tmp", + delete=False, + ) as handle: + temp_path = handle.name + json.dump(data, handle, indent=2, ensure_ascii=False) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temp_path, path) + finally: + if temp_path and os.path.exists(temp_path): + os.unlink(temp_path) diff --git a/src/osintai/temporal.py b/src/osintai/temporal.py new file mode 100644 index 0000000..5ae368f --- /dev/null +++ b/src/osintai/temporal.py @@ -0,0 +1,254 @@ +"""Temporal analysis over crawled material. + +The event stream combines crawl timestamps with dates found in page content, and every +event keeps its source URL. + +Original timestamps are preserved on every event. Normalization produces an additional +field; it never overwrites what the source said. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple + +from .provenance import ( + DERIVED, + HIGH, + LOW, + MEDIUM, + OBSERVED, + CheckResult, + Finding, + deterministic_confidence, + source_support, +) + +# Event kinds +FETCH = "page_fetched" +CONTENT_DATE = "date_in_content" + + +@dataclass +class Event: + """One dated thing, with the raw form it was written in and where it came from.""" + + when: datetime + kind: str + source: str + detail: str = "" + raw: str = "" + + def to_dict(self) -> Dict[str, Any]: + return { + "timestamp": self.when.isoformat(), + "date": self.when.date().isoformat(), + "kind": self.kind, + "source": self.source, + "detail": self.detail, + "raw": self.raw, + } + + +# Written date forms worth accepting. Model-reported `key_dates` are usually prose dates +# ("Feb 28, 2026"), and dropping them would leave the timeline built almost entirely from +# fetch timestamps, which say when OSINTai looked rather than when anything happened. +_DATE_FORMATS = ( + "%m/%d/%Y", "%d/%m/%Y", "%Y/%m/%d", + "%b %d, %Y", "%B %d, %Y", + "%b %d %Y", "%B %d %Y", + "%d %b %Y", "%d %B %Y", + "%d-%b-%Y", "%d-%B-%Y", + "%B %Y", "%b %Y", + "%Y-%m", "%Y", +) + + +def parse_timestamp(value: str) -> Optional[datetime]: + """Parse ISO-8601 and the common written date forms. Trailing Z is accepted.""" + text = (value or "").strip().rstrip(".,;") + if not text: + return None + iso_text = text[:-1] + "+00:00" if text.endswith("Z") else text + try: + parsed = datetime.fromisoformat(iso_text) + except ValueError: + for fmt in _DATE_FORMATS: + try: + parsed = datetime.strptime(text, fmt) + break + except ValueError: + continue + else: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed + + +def _plausible(when: datetime) -> bool: + """Reject dates a web crawl cannot meaningfully be reporting on.""" + now = datetime.now(timezone.utc) + return datetime(1990, 1, 1, tzinfo=timezone.utc) <= when <= now + timedelta(days=366 * 2) + + +def build_events( + page_records: Iterable[Dict[str, Any]], + content_dates: Optional[Dict[str, List[str]]] = None, +) -> Tuple[List[Event], List[str]]: + """Assemble the run's event stream. Returns events and non-fatal parse errors.""" + events: List[Event] = [] + errors: List[str] = [] + + for record in page_records or []: + url = record.get("url") + fetched_at = record.get("fetched_at") + if not url or not fetched_at: + continue + try: + when = datetime.fromtimestamp(float(fetched_at), tz=timezone.utc) + except (TypeError, ValueError, OSError) as exc: + errors.append(f"{url}: unreadable fetch timestamp {fetched_at!r}: {exc}") + continue + events.append(Event( + when=when, + kind=FETCH, + source=url, + detail=(record.get("title") or "")[:120], + raw=str(fetched_at), + )) + + for url, raw_dates in (content_dates or {}).items(): + for raw in raw_dates: + when = parse_timestamp(raw) + if when is None: + errors.append(f"{url}: could not parse date {raw!r}") + continue + if not _plausible(when): + continue + events.append(Event( + when=when, kind=CONTENT_DATE, source=url, + detail="date published in page content", raw=raw, + )) + + events.sort(key=lambda e: e.when) + return events, errors + + +def analyze_timeline(events: List[Event], gap_threshold_days: int = 90) -> CheckResult: + """Order events, measure gaps, and flag silence, bursts and the open trailing window.""" + result = CheckResult(check_name="Temporal Analysis") + + content_events = [e for e in events if e.kind == CONTENT_DATE] + if not events: + result.notes.append("No dated events available. Temporal analysis skipped.") + return result + + for index, event in enumerate(events): + gap = "" if index == 0 else (event.when - events[index - 1].when).days + row = event.to_dict() + row["gap_from_previous_days"] = gap + result.rows.append(row) + + if len(content_events) < 2: + result.notes.append( + f"{len(content_events)} content date(s) recovered; gap analysis needs at least 2." + ) + else: + for before, after in zip(content_events, content_events[1:]): + gap_days = (after.when - before.when).days + if gap_days <= gap_threshold_days: + continue + result.findings.append(Finding( + check="Activity Gap", + item=f"{before.when.date().isoformat()} to {after.when.date().isoformat()}", + reason=( + f"Chronological gap of {gap_days} day(s) between dated content, exceeding the " + f"{gap_threshold_days}-day threshold." + ), + next_step=( + "Determine whether this is genuine silence, content removed since publication, " + "activity that moved to another channel, or simply a limit of what this crawl " + "reached." + ), + origin=DERIVED, + priority=MEDIUM, + sources=[before.source, after.source], + evidence={ + "gap_start": before.when.date().isoformat(), + "gap_end": after.when.date().isoformat(), + "gap_days": gap_days, + "last_before": before.raw, + "first_after": after.raw, + }, + method="chronological_gap", + confidence=[deterministic_confidence( + 0.6, "gap in crawled content only; absence of records is not absence of activity")], + )) + + latest = content_events[-1] + trailing = (datetime.now(timezone.utc) - latest.when).days + if trailing > gap_threshold_days: + result.findings.append(Finding( + check="Open Trailing Gap", + item=f"{latest.when.date().isoformat()} to present", + reason=( + f"Most recent dated content is {trailing} day(s) old, exceeding the " + f"{gap_threshold_days}-day threshold." + ), + next_step=( + "Re-run collection against current sources to establish whether activity " + "continued outside what this run captured." + ), + origin=DERIVED, + priority=HIGH, + sources=[latest.source], + evidence={ + "last_dated_content": latest.when.date().isoformat(), + "days_since": trailing, + }, + method="trailing_gap", + confidence=[deterministic_confidence(0.6, "based on crawled content dates only")], + )) + + # Bursts: days carrying far more dated content than the active-day average. + per_day: Dict[str, List[Event]] = {} + for event in content_events: + per_day.setdefault(event.when.date().isoformat(), []).append(event) + if len(per_day) >= 3: + counts = [len(v) for v in per_day.values()] + mean = sum(counts) / len(counts) + for day, day_events in sorted(per_day.items()): + if len(day_events) >= max(3, mean * 3): + sources = sorted({e.source for e in day_events}) + result.findings.append(Finding( + check="Activity Burst", + item=day, + reason=( + f"{len(day_events)} dated items on one day against an active-day mean " + f"of {mean:.1f}." + ), + next_step="Establish what happened on this date and whether the items share a cause.", + origin=DERIVED, + priority=MEDIUM, + sources=sources[:25], + evidence={"day": day, "item_count": len(day_events), "mean": round(mean, 2)}, + method="daily_burst", + confidence=[deterministic_confidence(0.55, "frequency observation over this run"), + source_support(sources)], + )) + + first, last = events[0], events[-1] + result.stats = { + "event_count": len(events), + "content_date_count": len(content_events), + "window_start": first.when.isoformat(), + "window_end": last.when.isoformat(), + "window_days": (last.when - first.when).days, + } + result.notes.append( + f"Ordered {len(events)} event(s) spanning {result.stats['window_days']} day(s). " + f"Flagged {result.finding_count} temporal finding(s)." + ) + return result diff --git a/src/osintai/training_export.py b/src/osintai/training_export.py new file mode 100644 index 0000000..dbc6a29 --- /dev/null +++ b/src/osintai/training_export.py @@ -0,0 +1,201 @@ +"""Export evaluation and training material from a completed OSINT run. + +OSINTai does not train. There is no trainer, no adapter, no MLX, no LoRA, no GGUF conversion +and no training dependency anywhere in this package, and nothing here runs unless the +operator passes the export flag. + +Emitted files use portable JSON and JSONL formats: + + tasks.jsonl {id, type, prompt, rubric} evaluation task battery + preferences.jsonl {prompt, rejected, chosen, ...} preference pairs for later review + scored.jsonl per-analysis rubric scores + manifest.json provenance of the export + +`chosen` is deliberately empty. Generating the corrected answer requires a model rewriting +another model's failure, which belongs in the training pipeline and not in a collection run. +""" + +from __future__ import annotations + +import json +import os +import time +from typing import Any, Callable, Dict, List, Optional + +from .evaluation import RUBRIC_WEIGHTS, weak_examples +from .provenance import CheckResult +from .storage import safe_mkdir, write_json + +EXPORT_DIRNAME = "training_export" + +# Stable task categories for downstream evaluation tooling. +TASK_ENTITY = "entity_extraction" +TASK_CONFIDENCE = "confidence_language" +TASK_SOURCE = "source_discipline" +TASK_PIVOT = "pivot_reasoning" +TASK_FORMAT = "format_compliance" + +_FAILURE_TASK_TYPE = { + "entity_accuracy": TASK_ENTITY, + "confidence_language": TASK_CONFIDENCE, + "source_discipline": TASK_SOURCE, + "pivot_reasoning": TASK_PIVOT, + "format_compliance": TASK_FORMAT, +} + + +def _write_jsonl(path: str, rows: List[Dict[str, Any]]) -> str: + with open(path, "w", encoding="utf-8") as handle: + for row in rows: + handle.write(json.dumps(row, ensure_ascii=False, default=str) + "\n") + return path + + +def _weakest_dimension(scores: Dict[str, float]) -> str: + return min(scores, key=lambda k: scores[k]) if scores else "entity_accuracy" + + +def build_tasks( + evaluations: List[Dict[str, Any]], + prompt_loader: Callable[[str], str], + limit: int = 100, +) -> List[Dict[str, Any]]: + """Build an evaluation battery from real pages this run struggled on. + + Real crawled pages make harder and more representative eval tasks than hand-written + prompts, which makes the export useful for downstream evaluation. + """ + tasks: List[Dict[str, Any]] = [] + for index, evaluation in enumerate(evaluations[:limit], start=1): + url = evaluation.get("url", "") + prompt = prompt_loader(url) + if not prompt: + continue + scores = evaluation.get("scores", {}) + dimension = _weakest_dimension(scores) + tasks.append({ + "id": f"OSINTAI-{index:04d}", + "type": _FAILURE_TASK_TYPE.get(dimension, TASK_ENTITY), + "prompt": prompt, + "rubric": { + "weakest_dimension": dimension, + "observed_score": scores.get(dimension), + "observed_weighted_score": evaluation.get("weighted_score"), + "failures": evaluation.get("failures", []), + "confidence_required": True, + "source_attribution_required": True, + }, + "source_url": url, + "origin": "osintai_production_run", + }) + return tasks + + +def build_preference_pairs( + evaluations: List[Dict[str, Any]], + analyses_by_url: Dict[str, Dict[str, Any]], + prompt_loader: Callable[[str], str], + threshold: float = 0.6, + limit: int = 100, +) -> List[Dict[str, Any]]: + """DPO pairs from analyses that scored badly against the rubric. + + The failing output becomes `rejected` with the specific failures attached. `chosen` is + left empty for a separate, reviewed training workflow to construct. + """ + pairs: List[Dict[str, Any]] = [] + for evaluation in evaluations: + if evaluation.get("weighted_score", 1.0) >= threshold: + continue + url = evaluation.get("url", "") + analysis = analyses_by_url.get(url) + prompt = prompt_loader(url) + if not analysis or not prompt: + continue + pairs.append({ + "task_id": f"OSINTAI-DPO-{len(pairs) + 1:04d}", + "task_type": _FAILURE_TASK_TYPE.get( + _weakest_dimension(evaluation.get("scores", {})), TASK_ENTITY + ), + "prompt": prompt, + "rejected": json.dumps(analysis, ensure_ascii=False), + "chosen": "", + "chosen_status": "PENDING_REVIEW", + "failures_addressed": evaluation.get("failures", []), + "original_score": evaluation.get("weighted_score"), + "dimension_scores": evaluation.get("scores", {}), + "source_url": url, + "model": evaluation.get("model", ""), + }) + if len(pairs) >= limit: + break + return pairs + + +def export( + run_dir: str, + evaluation_result: CheckResult, + analyses_by_url: Dict[str, Dict[str, Any]], + prompt_loader: Callable[[str], str], + model: str = "", + run_id: str = "", + threshold: float = 0.6, +) -> Dict[str, Any]: + """Write the handoff package. Returns a summary of what was written. + + This is an export. It reads run artifacts and writes files. It does not fine-tune, does + not modify a model, and does not contact anything. + """ + out_dir = os.path.join(run_dir, EXPORT_DIRNAME) + safe_mkdir(out_dir) + + evaluations = [r for r in evaluation_result.rows if isinstance(r, dict)] + weak = weak_examples(evaluation_result, threshold=threshold) + + tasks = build_tasks(weak or evaluations, prompt_loader) + pairs = build_preference_pairs( + evaluations, analyses_by_url, prompt_loader, threshold=threshold + ) + + tasks_path = _write_jsonl(os.path.join(out_dir, "tasks.jsonl"), tasks) + pairs_path = _write_jsonl(os.path.join(out_dir, "preferences.jsonl"), pairs) + scored_path = _write_jsonl(os.path.join(out_dir, "scored.jsonl"), evaluations) + + manifest = { + "produced_by": "OSINTai", + "purpose": "training and evaluation dataset export", + "run_id": run_id, + "run_dir": run_dir, + "exported_at": time.strftime("%Y-%m-%dT%H:%M:%S"), + "analysis_model": model, + "rubric_weights": RUBRIC_WEIGHTS, + "weak_threshold": threshold, + "counts": { + "analyses_evaluated": len(evaluations), + "weak_analyses": len(weak), + "eval_tasks": len(tasks), + "preference_pairs": len(pairs), + }, + "quality_summary": evaluation_result.stats, + "files": { + "tasks": os.path.basename(tasks_path), + "preferences": os.path.basename(pairs_path), + "scored": os.path.basename(scored_path), + }, + "notes": [ + "Scoring is deterministic and computed against page text; no model graded another model.", + "Preference pairs carry `rejected` only. `chosen` requires a separate reviewed workflow.", + "OSINTai performs no training. This package is an inert local export.", + ], + } + manifest_path = os.path.join(out_dir, "manifest.json") + write_json(manifest_path, manifest) + + return { + "dir": out_dir, + "tasks": tasks_path, + "preferences": pairs_path, + "scored": scored_path, + "manifest": manifest_path, + "counts": manifest["counts"], + } diff --git a/tests/test_analysis_layer.py b/tests/test_analysis_layer.py new file mode 100644 index 0000000..47eccd1 --- /dev/null +++ b/tests/test_analysis_layer.py @@ -0,0 +1,819 @@ +"""Regression and capability tests for the OSINTai analysis layer.""" + +import json +import os +import sys +import tempfile +import time +import unittest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src")) + +from osintai import correlation, entities, evaluation, hypotheses, patterns # noqa: E402 +from osintai import multimodel, pivots, prompts, temporal # noqa: E402 +from osintai.analyzer import compute_page_signal # noqa: E402 +from osintai.pipeline import AnalysisOptions, analyze_run # noqa: E402 +from osintai.provenance import ( # noqa: E402 + DERIVED, + HIGH, + HYPOTHESIS, + MODEL, + OBSERVED, + CheckResult, + Confidence, + Finding, + Hypothesis, + cross_model_confidence, + sort_findings, + source_support, +) +from osintai.report import write_analysis_report, write_report # noqa: E402 +from osintai.storage import append_jsonl, safe_mkdir, sha1 # noqa: E402 + + +def _read(path: str) -> str: + """Read a whole file and close it. Keeps the suite free of ResourceWarnings.""" + with open(path, encoding="utf-8") as handle: + return handle.read() + + +# --------------------------------------------------------------------------- +# Regression: existing behavior +# --------------------------------------------------------------------------- + +BASELINE_PROMPT = """ +You are an OSINT analyst. Produce a structured intelligence extraction from this webpage. + +Rules: +- No filler. No moralizing. +- Only use evidence from the content. +- Output valid JSON only. + +Return schema: +{ + "url": "...", + "title": "...", + "summary": "2-4 sentences", + "key_entities": ["..."], + "key_locations": ["..."], + "key_dates": ["..."], + "keywords": ["..."], + "risk_flags": ["..."], + "actionable_leads": ["..."] +} + +URL: https://one.test/page +TITLE: Example Title + +CONTENT: +Body text. +""".strip() + + +class PromptRegressionTests(unittest.TestCase): + def test_standard_prompt_is_byte_identical_to_baseline(self): + # osint-tuned-v3 was trained against this exact text. If this fails, the model is + # being sent something it was not tuned for. + self.assertEqual( + prompts.page_prompt("https://one.test/page", "Example Title", "Body text."), + BASELINE_PROMPT, + ) + + def test_crawler_prompt_delegates_without_changing_output(self): + from osintai.crawler import AsyncCrawler + + crawler = object.__new__(AsyncCrawler) + crawler.prompt_profile = prompts.STANDARD + self.assertEqual( + crawler._analysis_prompt("https://one.test/page", "Example Title", "Body text."), + BASELINE_PROMPT, + ) + + def test_threat_profile_is_a_different_prompt_with_the_same_schema(self): + threat = prompts.page_prompt("https://one.test/", "T", "Body", profile=prompts.THREAT) + self.assertNotEqual(threat, BASELINE_PROMPT) + for key in ("key_entities", "risk_flags", "actionable_leads", "summary"): + self.assertIn(key, threat) + + +class ScoringRegressionTests(unittest.TestCase): + def test_page_signal_formula_is_unchanged(self): + indicators = { + "emails": ["a@b.test"] * 12, # capped at 10 -> 20.0 + "phones": ["8165550142"] * 7, # capped at 5 -> 7.5 + "btc_addresses": ["x"] * 4, # capped at 3 -> 9.0 + "eth_addresses": ["y"] * 4, # capped at 3 -> 9.0 + "social_handles": ["@z"] * 8, # capped at 5 -> 5.0 + } + analysis = { + "risk_flags": ["a", "b"], # 10.0 + "actionable_leads": ["l1", "l2", "l3"], # 9.0 + "key_entities": ["e"] * 15, # capped at 10 -> 10.0 + "key_locations": ["loc"] * 9, # capped at 5 -> 7.5 + "keywords": ["k"] * 30, # capped at 20 -> 10.0 + } + self.assertEqual(compute_page_signal(indicators, analysis), 97.0) + + def test_page_signal_tolerates_empty_analysis(self): + self.assertEqual(compute_page_signal({}, {}), 0.0) + + +class ReportRegressionTests(unittest.TestCase): + def test_ranked_report_format_is_unchanged(self): + with tempfile.TemporaryDirectory() as run_dir: + path = write_report(run_dir, [ + {"url": "https://one.test/", "score": 12.5, "title": "T", + "summary": "S", "risk_flags": ["rf"]}, + ]) + content = _read(path) + self.assertTrue(content.startswith("OSINTai Report\n" + "=" * 60)) + self.assertIn("01. score=12.5 https://one.test/", content) + self.assertIn(" title: T", content) + self.assertIn(" summary: S", content) + self.assertIn(" risk_flags: rf", content) + self.assertTrue(os.path.exists(os.path.join(run_dir, "ranked_pages.json"))) + + +class ExtractorRegressionTests(unittest.TestCase): + def test_indicator_schema_keys_are_unchanged(self): + from osintai.extractor import Extractor + + expected = { + "url", "domain", "domains", "emails", "phones", "urls", "ip_addresses", + "btc_addresses", "eth_addresses", "social_handles", "email_count", + "phone_count", "url_count", "domain_count", "ip_count", "btc_count", + "eth_count", "social_count", + } + indicators = Extractor().extract_indicators("https://one.test/", "text", "") + self.assertEqual(set(indicators), expected) + + +class NoTrainingInOSINTaiTests(unittest.TestCase): + def test_no_training_dependency_or_entry_point_exists(self): + # Training must never run during OSINT analysis, and the heavyweight training + # stack must never become an OSINTai dependency. + src = os.path.join(os.path.dirname(__file__), "..", "src", "osintai") + forbidden = ("import mlx", "from mlx", "import torch", "lora_rank", "safetensors") + for name in os.listdir(src): + if not name.endswith(".py"): + continue + body = _read(os.path.join(src, name)) + for token in forbidden: + self.assertNotIn(token, body, f"{name} references training stack: {token}") + + requirements = _read( + os.path.join(os.path.dirname(__file__), "..", "requirements.txt") + ).lower() + for package in ("mlx", "torch", "transformers", "peft", "numpy"): + self.assertNotIn(package, requirements) + + +# --------------------------------------------------------------------------- +# Capability: provenance +# --------------------------------------------------------------------------- + +class ProvenanceTests(unittest.TestCase): + def test_finding_rejects_unknown_origin_and_priority(self): + with self.assertRaises(ValueError): + Finding(check="c", item="i", reason="r", next_step="n", origin="FACT") + with self.assertRaises(ValueError): + Finding(check="c", item="i", reason="r", next_step="n", priority="URGENT") + + def test_hypothesis_origin_cannot_be_relabelled_as_observed(self): + hypothesis = Hypothesis(statement="maybe") + hypothesis.origin = OBSERVED # even if reassigned... + payload = hypothesis.to_dict() + self.assertEqual(payload["origin"], HYPOTHESIS) # ...serialization stays honest + self.assertEqual(payload["label"], HYPOTHESIS) + + def test_confidence_kinds_stay_distinct_and_clamped(self): + with self.assertRaises(ValueError): + Confidence("vibes", 0.5) + self.assertEqual(Confidence("deterministic", 5.0).value, 1.0) + self.assertEqual(Confidence("deterministic", -3.0).value, 0.0) + + def test_source_support_scales_with_independent_sources(self): + self.assertEqual(source_support([]).value, 0.0) + one = source_support(["https://a.test/"]).value + three = source_support(["https://a.test/", "https://b.test/", "https://c.test/"]).value + self.assertLess(one, three) + self.assertLess(three, 1.0) # crawled corroboration never reaches certainty + + def test_findings_sort_high_priority_first(self): + low = Finding(check="b", item="1", reason="", next_step="", priority="LOW") + high = Finding(check="a", item="2", reason="", next_step="", priority=HIGH) + self.assertEqual([f.priority for f in sort_findings([low, high])], [HIGH, "LOW"]) + + +# --------------------------------------------------------------------------- +# Capability: entities +# --------------------------------------------------------------------------- + +class EntityTests(unittest.TestCase): + def test_identifier_typing(self): + self.assertEqual(entities.detect_type("j.doe@example.test"), entities.EMAIL) + self.assertEqual(entities.detect_type("(816) 555-0142"), entities.PHONE) + self.assertEqual(entities.detect_type("John Martinez"), entities.NAME) + self.assertEqual(entities.detect_type("h4ckerman"), entities.USERNAME) + + def test_one_number_written_four_ways_is_one_entity(self): + index = entities.EntityIndex() + for written in ("816-555-0142", "(816) 555-0142", "+18165550142", "816.555.0142"): + index.add(entities.PHONE, written, f"https://{written}.test/") + self.assertEqual(len(index.of_kind(entities.PHONE)), 1) + entity = index.of_kind(entities.PHONE)[0] + self.assertEqual(entity.canonical_value, "8165550142") + self.assertEqual(len(entity.sources), 4) + self.assertEqual(len(entity.observed_forms), 4) + + def test_observed_forms_are_separate_from_possible_variants(self): + index = entities.EntityIndex() + index.add(entities.PHONE, "816-555-0142", "https://a.test/") + entity = index.of_kind(entities.PHONE)[0] + self.assertEqual(entity.observed_forms, ["816-555-0142"]) + self.assertGreater(len(entity.variants), 1) # possible forms, not seen ones + + def test_extended_extraction_finds_new_indicator_classes(self): + extras = entities.extract_extended( + "Meeting on 2024-01-31 with John Martinez at 4421 Troost Ave.\n" + "api_key: AKIAIOSFODNN7EXAMPLEKEY123\n" + ) + self.assertIn("2024-01-31", extras["dates"]) + self.assertIn("John Martinez", extras["name_candidates"]) + self.assertTrue(any("4421 Troost Ave" in a for a in extras["addresses"])) + self.assertTrue(extras["api_tokens"]) + + def test_non_ascii_domains_and_handles_are_recovered(self): + # The crawler's own patterns are ASCII-only, so a Cyrillic lookalike never reaches + # the homoglyph check unless this recovers it. + extras = entities.extract_extended( + "Partner pаypal.test and handle @аdmin alongside example.test and @admin" + ) + self.assertIn("pаypal.test", extras["unicode_domains"]) + self.assertIn("@аdmin", extras["unicode_handles"]) + # Plain ASCII values are already in the crawler's output and are not duplicated. + self.assertNotIn("example.test", extras["unicode_domains"]) + self.assertNotIn("@admin", extras["unicode_handles"]) + + def test_bare_urls_are_not_read_as_credentials(self): + extras = entities.extract_extended( + "https://example.test/some/path\nhttp://other.test/thing\nNote: something long here\n" + ) + self.assertEqual(extras["credential_pairs"], []) + + def test_real_credential_shape_is_still_caught(self): + extras = entities.extract_extended("admin:hunter2secret\n") + self.assertEqual(len(extras["credential_pairs"]), 1) + + def test_multi_source_ranking(self): + index = entities.EntityIndex() + index.add(entities.EMAIL, "a@x.test", "https://1.test/") + for i in range(4): + index.add(entities.EMAIL, "b@x.test", f"https://{i}.test/") + top = index.multi_source(minimum=2) + self.assertEqual(top[0].canonical_value, "b@x.test") + + +# --------------------------------------------------------------------------- +# Capability: deterministic checks +# --------------------------------------------------------------------------- + +class PatternTests(unittest.TestCase): + def _index_with(self, kind, value): + index = entities.EntityIndex() + index.add(kind, value, "https://source.test/") + return index + + def test_cyrillic_homoglyph_domain_is_flagged_with_ascii_form(self): + index = self._index_with(entities.DOMAIN, "pаypal.test") # Cyrillic a + result = patterns.check_homoglyphs(index) + self.assertEqual(result.finding_count, 1) + finding = result.findings[0] + self.assertEqual(finding.origin, DERIVED) + self.assertEqual(finding.evidence["ascii_normalization"], "paypal.test") + + def test_zero_width_character_is_high_priority(self): + index = self._index_with(entities.USERNAME, "ad​min") + result = patterns.check_homoglyphs(index) + self.assertEqual(result.findings[0].priority, HIGH) + + def test_plain_ascii_produces_no_homoglyph_findings(self): + index = self._index_with(entities.DOMAIN, "example.test") + self.assertEqual(patterns.check_homoglyphs(index).finding_count, 0) + + def test_private_and_public_ip_classification(self): + for private in ("10.0.0.1", "192.168.1.1", "172.16.5.4", "127.0.0.1"): + self.assertTrue(patterns._is_private_ip(private), private) + for public in ("8.8.8.8", "172.32.0.1", "1.1.1.1"): + self.assertFalse(patterns._is_private_ip(public), public) + + def test_sensitive_and_internal_indicators_are_flagged(self): + index = entities.EntityIndex() + index.add(entities.EMAIL, "agent@agency.gov", "https://a.test/") + index.add(entities.DOMAIN, "base.mil", "https://a.test/") + index.add(entities.IP, "10.1.2.3", "https://a.test/") + checks = {f.check for f in patterns.check_sensitive_infrastructure(index).findings} + self.assertEqual( + checks, {"Sensitive Email Domain", "Sensitive Domain", "Internal IP Exposure"} + ) + + def test_secret_findings_never_record_the_secret_value(self): + extras = {"https://a.test/": entities.extract_extended( + "api_key: SUPERSECRETVALUE0123456789\nadmin:hunter2secret\n" + )} + result = patterns.check_secret_exposure(extras) + self.assertGreater(result.finding_count, 0) + serialized = json.dumps([f.to_dict() for f in result.findings]) + self.assertNotIn("SUPERSECRETVALUE0123456789", serialized) + self.assertNotIn("hunter2secret", serialized) + self.assertIn("value_recorded", serialized) + + def test_jwt_payload_is_decoded(self): + # {"iss":"acme","sub":"1234"} + # Assemble the fixture so scanners do not mistake test data for a live token. + token = ".".join( + ("eyJ" + "hbGciOiJIUzI1NiJ9", "eyJ" + "pc3MiOiJhY21lIiwic3ViIjoiMTIzNCJ9", "abcdefghij") + ) + claims = patterns.decode_jwt_payload(token) + self.assertEqual(claims["iss"], "acme") + result = patterns.check_secret_exposure({"https://a.test/": { + "api_tokens": [], "jwts": [token], "credential_pairs": [], + }}) + jwt_finding = next(f for f in result.findings if f.check == "JWT Present") + self.assertIn("iss", jwt_finding.evidence["claim_keys"]) + self.assertNotIn(token, json.dumps(jwt_finding.to_dict())) + + def test_malformed_jwt_returns_none(self): + self.assertIsNone(patterns.decode_jwt_payload("not.a.jwt")) + self.assertIsNone(patterns.decode_jwt_payload("onlyonepart")) + + def test_generated_content_fingerprint(self): + pages = [{"url": "https://a.test/"}, {"url": "https://b.test/"}] + texts = { + "https://a.test/": "As an AI language model, I cannot provide that information.", + "https://b.test/": "An ordinary article about municipal drainage.", + } + result = patterns.check_generated_text(pages, lambda u: texts.get(u, "")) + self.assertEqual(result.finding_count, 1) + self.assertEqual(result.findings[0].item, "https://a.test/") + + def test_outlier_detection_requires_enough_pages(self): + index = entities.EntityIndex() + result = patterns.check_recurring_and_outliers(index, [{"url": "a", "score": 1}]) + self.assertTrue(any("Fewer than 5" in n for n in result.notes)) + + +# --------------------------------------------------------------------------- +# Capability: correlation +# --------------------------------------------------------------------------- + +class CorrelationTests(unittest.TestCase): + def test_co_occurring_identifiers_link_with_evidence(self): + index = entities.EntityIndex() + for page in ("https://p1.test/", "https://p2.test/"): + index.add(entities.EMAIL, "target@x.test", page) + index.add(entities.USERNAME, "@targethandle", page) + result = correlation.correlate(index, [], page_count=2) + self.assertGreater(len(result.rows), 0) + row = result.rows[0] + self.assertEqual(row["status"], "CANDIDATE") + self.assertGreater(row["evidence_count"], 0) + + def test_identifiers_that_never_co_occur_are_not_linked(self): + index = entities.EntityIndex() + index.add(entities.EMAIL, "a@x.test", "https://p1.test/") + index.add(entities.USERNAME, "@unrelated", "https://p2.test/") + rows = correlation.correlate(index, [], page_count=2).rows + self.assertFalse([r for r in rows if r["relation"] == correlation.CO_OCCURS]) + + def test_index_position_does_not_create_links(self): + # Same-index values from different pages must not produce a relationship. + index = entities.EntityIndex() + index.add(entities.EMAIL, "first@x.test", "https://p1.test/") + index.add(entities.PHONE, "8165550142", "https://p2.test/") + index.add(entities.EMAIL, "second@x.test", "https://p3.test/") + index.add(entities.PHONE, "8165550143", "https://p4.test/") + rows = correlation.correlate(index, [], page_count=4).rows + self.assertEqual([r for r in rows if r["relation"] == correlation.CO_OCCURS], []) + + def test_site_wide_identifiers_are_excluded_from_pairing(self): + index = entities.EntityIndex() + pages = [f"https://p{i}.test/" for i in range(20)] + for page in pages: + index.add(entities.EMAIL, "footer@site.test", page) # on every page + index.add(entities.USERNAME, "@sitehandle", page) # on every page + index.add(entities.EMAIL, "rare@site.test", pages[0]) + rows = correlation.correlate(index, [], page_count=20).rows + pairs = [r for r in rows if r["relation"] == correlation.CO_OCCURS] + for row in pairs: + self.assertNotIn("footer@site.test", (row["left"]["value"], row["right"]["value"])) + + def test_email_local_part_matching_a_handle_is_a_candidate_not_an_identity(self): + index = entities.EntityIndex() + index.add(entities.EMAIL, "jmartinez@x.test", "https://p.test/") + index.add(entities.USERNAME, "@jmartinez", "https://p.test/") + rows = correlation.correlate(index, [], page_count=1).rows + match = next(r for r in rows if r["relation"] == correlation.LOCAL_PART_MATCH) + self.assertEqual(match["status"], "CANDIDATE") + self.assertLess(match["score"], 0.8) + self.assertIn("pivot, not an identity", match["rationale"]) + + def test_domain_families_group_subdomains(self): + index = entities.EntityIndex() + for host in ("mail.example.test", "vpn.example.test", "www.example.test"): + index.add(entities.DOMAIN, host, "https://p.test/") + rows = correlation.correlate(index, [], page_count=1).rows + self.assertTrue([r for r in rows if r["relation"] == correlation.SHARES_DOMAIN]) + + def test_domain_url_mapping_ranks_by_frequency(self): + rows = [ + {"url": "https://a.test/1", "domain": "a.test", + "urls": ["https://b.test/x", "https://b.test/y", "https://c.test/z"]}, + ] + mapping = correlation.map_domains_to_urls(rows) + self.assertEqual(mapping["top_domains"][0][0], "b.test") + + +# --------------------------------------------------------------------------- +# Capability: temporal +# --------------------------------------------------------------------------- + +class TemporalTests(unittest.TestCase): + def test_parses_iso_and_written_date_forms(self): + for value in ("2024-01-31", "2024-01-31T14:22:00Z", "01/31/2024", + "Feb 28, 2026", "28 February 2026", "March 2025"): + self.assertIsNotNone(temporal.parse_timestamp(value), value) + self.assertIsNone(temporal.parse_timestamp("sometime last spring")) + + def test_events_are_ordered_and_keep_their_raw_form(self): + records = [ + {"url": "https://b.test/", "fetched_at": 1_700_000_100, "title": "B"}, + {"url": "https://a.test/", "fetched_at": 1_700_000_000, "title": "A"}, + ] + events, errors = temporal.build_events(records, {"https://a.test/": ["2020-05-05"]}) + self.assertEqual(errors, []) + self.assertEqual([e.when for e in events], sorted(e.when for e in events)) + content = next(e for e in events if e.kind == temporal.CONTENT_DATE) + self.assertEqual(content.raw, "2020-05-05") + self.assertEqual(content.source, "https://a.test/") + + def test_gap_over_threshold_is_flagged_and_under_is_not(self): + records = [{"url": "https://a.test/", "fetched_at": time.time(), "title": "A"}] + dates = {"https://a.test/": ["2020-01-01", "2020-01-20", "2021-06-01"]} + events, _ = temporal.build_events(records, dates) + result = temporal.analyze_timeline(events, gap_threshold_days=90) + gaps = [f for f in result.findings if f.check == "Activity Gap"] + self.assertEqual(len(gaps), 1) + self.assertGreater(gaps[0].evidence["gap_days"], 90) + + def test_unparseable_dates_become_errors_not_exceptions(self): + records = [{"url": "https://a.test/", "fetched_at": time.time()}] + events, errors = temporal.build_events(records, {"https://a.test/": ["not a date"]}) + self.assertEqual(len(errors), 1) + self.assertTrue(events) + + def test_empty_event_stream_is_handled(self): + result = temporal.analyze_timeline([], gap_threshold_days=90) + self.assertEqual(result.finding_count, 0) + self.assertTrue(result.notes) + + +# --------------------------------------------------------------------------- +# Capability: hypotheses and leads +# --------------------------------------------------------------------------- + +class HypothesisAndLeadTests(unittest.TestCase): + def test_hypotheses_carry_confirmation_and_refutation_conditions(self): + finding = Finding( + check="Unicode / Homoglyph", item="pаypal.test", + reason="non-ASCII", next_step="compare", + evidence={"ascii_normalization": "paypal.test"}, + sources=["https://a.test/"], + ) + derived = hypotheses.from_findings([finding]) + self.assertEqual(len(derived), 1) + self.assertTrue(derived[0].would_confirm) + self.assertTrue(derived[0].would_refute) + self.assertEqual(derived[0].to_dict()["origin"], HYPOTHESIS) + + def test_model_hypotheses_are_labelled_with_their_model(self): + payload = {"hypotheses": [ + {"statement": "The sites share an operator.", "confidence": 0.8}, + ]} + results = hypotheses.from_model(payload, "test-model:latest", ["https://a.test/"]) + self.assertEqual(len(results), 1) + self.assertEqual(results[0].model, "test-model:latest") + self.assertEqual(results[0].to_dict()["label"], HYPOTHESIS) + self.assertEqual(results[0].confidence[0].kind, "model_self") + + def test_model_hypotheses_from_garbage_payload(self): + self.assertEqual(hypotheses.from_model({"nope": 1}, "m", []), []) + self.assertEqual(hypotheses.from_model({"hypotheses": "text"}, "m", []), []) + + def test_leads_are_generated_per_identifier_type_with_risk_notes(self): + index = entities.EntityIndex() + index.add(entities.EMAIL, "j@x.test", "https://p.test/") + index.add(entities.USERNAME, "@jhandle", "https://p.test/") + index.add(entities.PHONE, "8165550142", "https://p.test/") + index.add(entities.DOMAIN, "x.test", "https://p.test/") + leads = pivots.generate_leads(index) + kinds = {lead.seed_type for lead in leads} + self.assertEqual( + kinds, {entities.EMAIL, entities.USERNAME, entities.PHONE, entities.DOMAIN} + ) + for lead in leads: + self.assertTrue(lead.target.startswith("http")) + self.assertTrue(lead.false_positive_risk) + self.assertEqual(lead.to_dict()["origin"], DERIVED) + + def test_infrastructure_pivots_are_built_for_domains(self): + labels = {p["label"] for p in pivots.pivots_for_domain("example.test")} + self.assertIn("WHOIS registration", labels) + self.assertIn("Certificate transparency", labels) + + +# --------------------------------------------------------------------------- +# Capability: multi-model and confidence +# --------------------------------------------------------------------------- + +class MultiModelTests(unittest.TestCase): + def _check(self, *verdicts): + return multimodel.CrossCheck( + claim="A claim", sources=["https://a.test/"], + verdicts=[multimodel.ModelVerdict(model=f"m{i}", verdict=v) + for i, v in enumerate(verdicts)], + ) + + def test_opposed_verdicts_are_contradictory_not_averaged(self): + check = self._check(multimodel.SUPPORTED, multimodel.CONTRADICTED) + self.assertEqual(check.agreement, "contradictory") + result = multimodel.summarize_cross_checks([check]) + finding = result.findings[0] + self.assertEqual(finding.check, "Model Disagreement") + self.assertEqual(finding.priority, HIGH) + self.assertEqual(finding.origin, MODEL) + self.assertIn("no model's verdict has been adopted", finding.next_step) + + def test_unanimous_agreement_is_not_called_verification(self): + check = self._check(multimodel.SUPPORTED, multimodel.SUPPORTED) + self.assertEqual(check.agreement, "unanimous") + finding = multimodel.summarize_cross_checks([check]).findings[0] + self.assertIn("not verification", finding.next_step) + + def test_even_split_has_no_majority(self): + check = self._check(multimodel.SUPPORTED, multimodel.INSUFFICIENT) + self.assertIsNone(check.majority) + + def test_a_failed_model_does_not_lose_the_other_verdicts(self): + check = multimodel.CrossCheck(claim="c", verdicts=[ + multimodel.ModelVerdict(model="a", verdict=multimodel.SUPPORTED), + multimodel.ModelVerdict(model="b", verdict="", error="connection refused"), + multimodel.ModelVerdict(model="c", verdict=multimodel.SUPPORTED), + ]) + self.assertEqual(len(check.answered), 2) + self.assertEqual(check.agreement, "unanimous") + + def test_unrecognized_verdict_is_an_error_not_a_vote(self): + verdict = multimodel._parse_verdict("m", {"verdict": "probably", "confidence": 0.9}) + self.assertTrue(verdict.error) + self.assertEqual(verdict.verdict, "") + + def test_cross_model_confidence_requires_more_than_one_model(self): + self.assertEqual(cross_model_confidence(1, 1).value, 0.0) + self.assertEqual(cross_model_confidence(2, 2).value, 1.0) + + def test_no_cross_check_requested_reports_cleanly(self): + result = multimodel.summarize_cross_checks([]) + self.assertEqual(result.finding_count, 0) + self.assertTrue(result.notes) + + +# --------------------------------------------------------------------------- +# Capability: evaluation +# --------------------------------------------------------------------------- + +class EvaluationTests(unittest.TestCase): + def test_ungrounded_entities_lower_entity_accuracy(self): + grounded, _ = evaluation.score_entity_accuracy( + {"key_entities": ["Acme Corp"]}, "A report about Acme Corp operations." + ) + invented, failures = evaluation.score_entity_accuracy( + {"key_entities": ["Acme Corp", "Nonexistent Holdings"]}, + "A report about Acme Corp operations.", + ) + self.assertEqual(grounded, 1.0) + self.assertLess(invented, 1.0) + self.assertTrue(failures) + + def test_overstated_language_is_penalized(self): + confident, failures = evaluation.score_confidence_language( + {"summary": "This confirmed and verified report proves the subject is guilty."} + ) + hedged, _ = evaluation.score_confidence_language( + {"summary": "The page appears to indicate a possible connection, unverified."} + ) + self.assertLess(confident, hedged) + self.assertTrue(failures) + + def test_missing_schema_keys_lower_format_compliance(self): + complete, _ = evaluation.score_format_compliance( + {k: ([] if k in evaluation.LIST_KEYS else "x") for k in evaluation.REQUIRED_KEYS} + ) + partial, failures = evaluation.score_format_compliance({"url": "u"}) + self.assertEqual(complete, 1.0) + self.assertLess(partial, complete) + self.assertTrue(failures) + + def test_vague_leads_lower_pivot_reasoning(self): + vague, _ = evaluation.score_pivot_reasoning({"actionable_leads": ["investigate further"]}) + specific, _ = evaluation.score_pivot_reasoning( + {"actionable_leads": ["Run WHOIS on example.test and compare the registrant"]} + ) + self.assertLess(vague, specific) + + def test_weights_match_the_release_rubric_and_sum_to_one(self): + self.assertEqual(evaluation.RUBRIC_WEIGHTS["entity_accuracy"], 0.30) + self.assertEqual(evaluation.RUBRIC_WEIGHTS["confidence_language"], 0.25) + self.assertAlmostEqual(sum(evaluation.RUBRIC_WEIGHTS.values()), 1.0) + + def test_run_evaluation_aggregates_failures(self): + analyses = [ + {"url": "https://a.test/", "summary": "This confirmed report proves everything.", + "key_entities": ["Invented Entity"], "actionable_leads": ["investigate further"]}, + ] + result = evaluation.evaluate_run(analyses, lambda u: "unrelated page text", model="m") + self.assertEqual(result.stats["evaluated"], 1) + self.assertTrue(result.stats["dominant_failures"]) + self.assertLess(result.stats["mean_weighted_score"], 0.9) + + def test_evaluation_is_deterministic_across_calls(self): + analysis = {"url": "https://a.test/", "summary": "Appears possible.", + "key_entities": ["Acme"], "actionable_leads": ["Check WHOIS for acme.test"]} + first = evaluation.evaluate_analysis(analysis, "Acme page") + second = evaluation.evaluate_analysis(analysis, "Acme page") + self.assertEqual(first["weighted_score"], second["weighted_score"]) + self.assertEqual(first["method"], "deterministic_rubric") + + +# --------------------------------------------------------------------------- +# Capability: pipeline end to end +# --------------------------------------------------------------------------- + +def _build_run(run_dir: str) -> None: + """A minimal but realistic run directory.""" + safe_mkdir(os.path.join(run_dir, "pages_text")) + safe_mkdir(os.path.join(run_dir, "analysis")) + now = time.time() + + pages = [ + ("https://site.test/contact", "Contact", "Reach John Martinez at 816-555-0142. " + "Email jmartinez@site.test. Posted 2020-01-15."), + ("https://site.test/about", "About", "Contact (816) 555-0142 or @jmartinez. " + "Updated 2021-08-01. Internal host 10.0.0.5."), + ("https://site.test/notice", "Notice", "As an AI language model, I cannot help with that."), + ] + for url, title, text in pages: + with open(os.path.join(run_dir, "pages_text", f"{sha1(url)}.txt"), "w", + encoding="utf-8") as handle: + handle.write(text) + append_jsonl(os.path.join(run_dir, "urls_crawled.jsonl"), { + "url": url, "status_code": 200, "fetched_at": now, "title": title, + "text_len": len(text), + }) + append_jsonl(os.path.join(run_dir, "indicators.jsonl"), { + "url": url, "domain": "site.test", "domains": ["site.test", "pаypal.test"], + "emails": ["jmartinez@site.test", "agent@agency.gov"], + "phones": ["816-555-0142"] if "contact" in url else ["8165550142"], + "urls": ["https://other.test/x"], "ip_addresses": ["10.0.0.5"], + "btc_addresses": [], "eth_addresses": [], "social_handles": ["@jmartinez"], + }) + append_jsonl(os.path.join(run_dir, "page_scores.jsonl"), { + "url": url, "title": title, "score": 10.0, "summary": f"Summary of {title}", + "risk_flags": [], + }) + with open(os.path.join(run_dir, "analysis", f"{sha1(url)}.analysis.json"), "w", + encoding="utf-8") as handle: + json.dump({ + "url": url, "title": title, "summary": "The page appears to list contacts.", + "key_entities": ["John Martinez"], "key_locations": [], "key_dates": ["Feb 28, 2026"], + "keywords": ["contact"], "risk_flags": [], + "actionable_leads": ["Check WHOIS for site.test"], + }, handle) + + +class PipelineTests(unittest.TestCase): + def test_full_offline_analysis_produces_every_artifact(self): + with tempfile.TemporaryDirectory() as run_dir: + _build_run(run_dir) + output = analyze_run( + run_dir, + AnalysisOptions(evaluate=True, use_ollama=False, model="test-model"), + ollama=None, run_id="t1", log=lambda m: None, + ) + self.assertEqual(output.errors, []) + for name in ("findings.jsonl", "hypotheses.jsonl", "leads.jsonl", + "correlations.jsonl", "timeline.jsonl", "analysis_summary.json"): + self.assertTrue(os.path.exists(os.path.join(run_dir, name)), name) + + checks = {f.check for f in output.findings} + self.assertIn("Unicode / Homoglyph", checks) # Cyrillic domain + self.assertIn("Sensitive Email Domain", checks) # .gov address + self.assertIn("Internal IP Exposure", checks) # RFC1918 + self.assertIn("Generated-Content Fingerprint", checks) + self.assertTrue(output.hypotheses) + self.assertTrue(output.leads) + + def test_analysis_runs_with_ollama_disabled(self): + with tempfile.TemporaryDirectory() as run_dir: + _build_run(run_dir) + output = analyze_run( + run_dir, AnalysisOptions(use_ollama=False, deep=True, cross_check_models=["x"]), + ollama=None, run_id="t2", log=lambda m: None, + ) + # deep and cross-check are silently inert without a model; nothing crashes. + self.assertEqual(output.errors, []) + self.assertTrue(output.findings) + + def test_empty_run_directory_is_reported_not_crashed(self): + with tempfile.TemporaryDirectory() as run_dir: + output = analyze_run( + run_dir, AnalysisOptions(use_ollama=False), ollama=None, log=lambda m: None + ) + self.assertTrue(output.errors) + self.assertEqual(output.findings, []) + + def test_a_failing_stage_does_not_abort_the_pipeline(self): + from osintai import pipeline as pipeline_module + + original = pipeline_module.patterns.check_homoglyphs + pipeline_module.patterns.check_homoglyphs = lambda index: (_ for _ in ()).throw( + RuntimeError("deliberate stage failure") + ) + try: + with tempfile.TemporaryDirectory() as run_dir: + _build_run(run_dir) + output = analyze_run( + run_dir, AnalysisOptions(use_ollama=False), + ollama=None, log=lambda m: None, + ) + self.assertTrue(any("deliberate stage failure" in e for e in output.errors)) + self.assertTrue(output.findings) # every other stage still ran + finally: + pipeline_module.patterns.check_homoglyphs = original + + def test_training_export_requires_evaluation(self): + with tempfile.TemporaryDirectory() as run_dir: + _build_run(run_dir) + output = analyze_run( + run_dir, AnalysisOptions(use_ollama=False, training_export=True), + ollama=None, log=lambda m: None, + ) + self.assertTrue(any("requires --evaluate" in e for e in output.errors)) + self.assertNotIn("training_export", output.artifacts) + + def test_training_export_writes_portable_formats(self): + with tempfile.TemporaryDirectory() as run_dir: + _build_run(run_dir) + output = analyze_run( + run_dir, + AnalysisOptions(use_ollama=False, evaluate=True, training_export=True, + model="osint-tuned-v3:latest"), + ollama=None, run_id="t3", log=lambda m: None, + ) + export_dir = output.artifacts.get("training_export") + self.assertTrue(export_dir and os.path.isdir(export_dir)) + manifest = json.loads(_read(os.path.join(export_dir, "manifest.json"))) + self.assertEqual(manifest["produced_by"], "OSINTai") + self.assertEqual(manifest["rubric_weights"], evaluation.RUBRIC_WEIGHTS) + with open(os.path.join(export_dir, "tasks.jsonl"), encoding="utf-8") as handle: + for line in handle: + task = json.loads(line) + self.assertEqual(set(task) >= {"id", "type", "prompt", "rubric"}, True) + + def test_analysis_report_separates_observed_from_derived(self): + with tempfile.TemporaryDirectory() as run_dir: + _build_run(run_dir) + output = analyze_run( + run_dir, AnalysisOptions(use_ollama=False), ollama=None, log=lambda m: None + ) + path = write_analysis_report(run_dir, output, run_id="t4", scope={"seeds": 1}) + content = _read(path) + self.assertIn("OBSERVED — PRESENT IN SOURCE MATERIAL", content) + self.assertIn("DERIVED — COMPUTED FROM SOURCE MATERIAL", content) + self.assertIn("MODEL-ASSISTED — LANGUAGE MODEL INTERPRETATION", content) + self.assertIn("HYPOTHESES — NOT FINDINGS, NOT FACTS", content) + self.assertIn("RECOMMENDED NEXT STEPS", content) + + def test_analysis_does_not_modify_crawl_artifacts(self): + with tempfile.TemporaryDirectory() as run_dir: + _build_run(run_dir) + crawl_files = ["urls_crawled.jsonl", "indicators.jsonl", "page_scores.jsonl"] + before = {name: _read(os.path.join(run_dir, name)) for name in crawl_files} + analyze_run(run_dir, AnalysisOptions(use_ollama=False), ollama=None, + log=lambda m: None) + for name in crawl_files: + self.assertEqual(_read(os.path.join(run_dir, name)), before[name]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_seed_and_fetch_policy.py b/tests/test_seed_and_fetch_policy.py new file mode 100644 index 0000000..d08c88b --- /dev/null +++ b/tests/test_seed_and_fetch_policy.py @@ -0,0 +1,153 @@ +import os +import sys +import tempfile +import unittest + +import httpx + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src")) + +from osintai.cli import _resolve_seed_urls, _validate_run_id # noqa: E402 +from osintai.crawler import AsyncCrawler # noqa: E402 +from osintai.extractor import Extractor # noqa: E402 +from osintai.fetcher import AsyncFetcher, FetchRejected # noqa: E402 + + +class SeedInputTests(unittest.TestCase): + def test_explicit_seed_file_is_loaded_and_deduplicated(self): + with tempfile.TemporaryDirectory() as directory: + path = os.path.join(directory, "seeds.txt") + with open(path, "w", encoding="utf-8") as handle: + handle.write("# targets\nhttps://one.test/\nhttps://two.test/\nhttps://one.test/\n") + self.assertEqual( + _resolve_seed_urls(None, path, directory), + ["https://one.test/", "https://two.test/"], + ) + + def test_file_passed_as_seed_gets_actionable_error(self): + with tempfile.NamedTemporaryFile() as handle: + with self.assertRaisesRegex(ValueError, "--seed-file"): + _resolve_seed_urls([handle.name], None, os.getcwd()) + + def test_same_domain_scope_accepts_every_seed_host_only(self): + crawler = object.__new__(AsyncCrawler) + crawler.same_domain_only = True + crawler.allowed_seed_hosts = {"one.test", "two.test"} + self.assertTrue(crawler._scoped("https://one.test/page")) + self.assertTrue(crawler._scoped("https://two.test/page")) + self.assertFalse(crawler._scoped("https://outside.test/page")) + self.assertFalse(crawler._scoped("ftp://one.test/archive")) + self.assertFalse(crawler._scoped("https://user:secret@one.test/private")) + + def test_seed_urls_reject_embedded_credentials(self): + with self.assertRaisesRegex(ValueError, "invalid HTTP\\(S\\) seed URL"): + _resolve_seed_urls(["https://user:secret@one.test/"], None, os.getcwd()) + + def test_run_id_cannot_escape_the_run_directory(self): + for value in ("../outside", "nested/run", "/absolute", "..", ""): + with self.subTest(value=value), self.assertRaises(ValueError): + _validate_run_id(value) + self.assertEqual(_validate_run_id("case-2026.08_14"), "case-2026.08_14") + + +class ExtractorSafetyTests(unittest.TestCase): + def test_malformed_ipv6_like_url_is_discarded_without_crashing(self): + indicators = Extractor().extract_indicators( + "https://one.test/", + "Malformed evidence URL: http://[not-an-ipv6/path and https://valid.test/report.", + "", + ) + self.assertEqual(indicators["urls"], ["https://valid.test/report"]) + self.assertIn("valid.test", indicators["domains"]) + + +class FetchPolicyTests(unittest.IsolatedAsyncioTestCase): + async def test_html_within_limit_is_returned(self): + transport = httpx.MockTransport( + lambda request: httpx.Response( + 200, headers={"content-type": "text/html"}, text="safe" + ) + ) + fetcher = AsyncFetcher([], min_delay_s=0, max_delay_s=0, max_response_bytes=100) + async with httpx.AsyncClient(transport=transport) as client: + response = await fetcher.get(client, "https://one.test/") + self.assertEqual(response.text, "safe") + + async def test_binary_content_is_rejected(self): + transport = httpx.MockTransport( + lambda request: httpx.Response( + 200, headers={"content-type": "application/zip"}, content=b"PK" + ) + ) + fetcher = AsyncFetcher([], min_delay_s=0, max_delay_s=0) + async with httpx.AsyncClient(transport=transport) as client: + with self.assertRaisesRegex(FetchRejected, "not HTML/XHTML"): + await fetcher.get(client, "https://one.test/archive.zip") + + async def test_oversized_html_is_rejected(self): + transport = httpx.MockTransport( + lambda request: httpx.Response( + 200, headers={"content-type": "text/html"}, content=b"x" * 11 + ) + ) + fetcher = AsyncFetcher([], min_delay_s=0, max_delay_s=0, max_response_bytes=10) + async with httpx.AsyncClient(transport=transport) as client: + with self.assertRaisesRegex(FetchRejected, "byte limit"): + await fetcher.get(client, "https://one.test/") + + async def test_out_of_scope_redirect_is_rejected_before_request(self): + requested = [] + + def handler(request): + requested.append(str(request.url)) + return httpx.Response(302, headers={"location": "https://outside.test/secret"}) + + transport = httpx.MockTransport(handler) + fetcher = AsyncFetcher([], min_delay_s=0, max_delay_s=0) + validator = lambda url: httpx.URL(url).host == "one.test" + async with httpx.AsyncClient(transport=transport) as client: + with self.assertRaisesRegex(FetchRejected, "outside authorized scope"): + await fetcher.get( + client, "https://one.test/", redirect_validator=validator + ) + self.assertEqual(requested, ["https://one.test/"]) + + async def test_in_scope_redirect_is_followed(self): + requested = [] + + def handler(request): + requested.append(str(request.url)) + if request.url.path == "/": + return httpx.Response(302, headers={"location": "/final"}) + return httpx.Response( + 200, headers={"content-type": "text/html"}, text="ok" + ) + + transport = httpx.MockTransport(handler) + fetcher = AsyncFetcher([], min_delay_s=0, max_delay_s=0) + validator = lambda url: httpx.URL(url).host == "one.test" + async with httpx.AsyncClient(transport=transport) as client: + response = await fetcher.get( + client, "https://one.test/", redirect_validator=validator + ) + self.assertEqual(str(response.url), "https://one.test/final") + self.assertEqual(requested, ["https://one.test/", "https://one.test/final"]) + + +class OllamaEndpointTests(unittest.TestCase): + def test_ollama_endpoint_is_loopback_only(self): + from osintai.ollama_api import OllamaAPI + + self.assertEqual(OllamaAPI("http://127.0.0.1:11434").base_url, "http://127.0.0.1:11434") + self.assertEqual(OllamaAPI("http://[::1]:11434").base_url, "http://[::1]:11434") + for value in ( + "https://models.example/api", + "http://192.168.1.10:11434", + "http://user:pass@localhost:11434", + ): + with self.subTest(value=value), self.assertRaises(ValueError): + OllamaAPI(value) + + +if __name__ == "__main__": + unittest.main()