diff --git a/src/semble/cache.py b/src/semble/cache.py index d53ca666..bdf09b53 100644 --- a/src/semble/cache.py +++ b/src/semble/cache.py @@ -24,15 +24,19 @@ from semble.index import SembleIndex -def find_index_from_cache_folder(path: str) -> Path: - """Finds an index from a cache folder and a project path.""" +def cache_key(path: str) -> str: + """Compute the sha256 cache key for a local path or git URL.""" if is_git_url(path): data = path.encode("utf-8") else: normalized = Path(path).expanduser().resolve() data = str(normalized).encode("utf-8") - subdir_path = hashlib.new("sha256", data).hexdigest() - cache_dir = resolve_cache_folder() / subdir_path + return hashlib.new("sha256", data).hexdigest() + + +def find_index_from_cache_folder(path: str) -> Path: + """Finds an index from a cache folder and a project path.""" + cache_dir = resolve_cache_folder() / cache_key(path) return cache_dir / "index" diff --git a/src/semble/cli.py b/src/semble/cli.py index c89c225d..dad4770d 100644 --- a/src/semble/cli.py +++ b/src/semble/cli.py @@ -5,12 +5,13 @@ import sys import warnings from importlib.util import find_spec +from pathlib import Path from shutil import rmtree from typing import Literal from model2vec.utils import get_package_extras -from semble.cache import find_index_from_cache_folder, resolve_cache_folder +from semble.cache import cache_key, find_index_from_cache_folder, resolve_cache_folder from semble.index import SembleIndex from semble.index.types import PersistencePath from semble.installer.agents import AGENTS, IntegrationType @@ -22,7 +23,7 @@ _CLI_DISPATCH_ARGS = frozenset( {"search", "find-related", "install", "uninstall", "savings", "-h", "--help", "clear", "--version", "-V"} ) -_CLEAR_CHOICE = Literal["all", "index", "savings"] +_CLEAR_CHOICE = Literal["all", "index", "savings", "orphans"] _SHA_256_REGEX = re.compile(r"^[a-f0-9]{64}$") @@ -138,33 +139,70 @@ def _run_find_related( _maybe_save_index(index, path) +def _clear_indexes(cache_folder: Path) -> None: + """Remove all valid index entries from the cache folder.""" + indexes = [] + for path in cache_folder.glob("*/index"): + if not _SHA_256_REGEX.match(path.parent.name): + continue + if PersistencePath.from_path(path).non_existing(): + continue + indexes.append(path) + + if not indexes: + print(f"No indexes found to clear in `{cache_folder}`") + else: + for path in indexes: + index_folder = path.parent + rmtree(index_folder) + print(f"Cleared index at `{index_folder}`") + + +def _clear_savings(cache_folder: Path) -> None: + """Remove the savings file from the cache folder.""" + path = cache_folder / "savings.jsonl" + if not path.exists(): + print(f"No savings file found at `{path}`") + else: + path.unlink() + print(f"Cleared savings at `{path}`") + + +def _clear_orphans(cache_folder: Path) -> None: + """Remove index entries whose local root_path no longer exists.""" + orphans = [] + for path in cache_folder.glob("*/index"): + if not _SHA_256_REGEX.match(path.parent.name): + continue + try: + with open(path / "metadata.json", encoding="utf-8") as f: + metadata = json.load(f) + root_path = metadata.get("root_path") if isinstance(metadata, dict) else None + except (OSError, json.JSONDecodeError): + continue + # Git-URL entries store their temp clone dir as root_path, so only trust entries whose key matches. + if not isinstance(root_path, str) or not root_path or cache_key(root_path) != path.parent.name: + continue + if not Path(root_path).exists(): + orphans.append((path.parent, root_path)) + + if not orphans: + print("No orphaned indexes found") + else: + for index_folder, root_path in orphans: + rmtree(index_folder) + print(f"Cleared orphaned index for `{root_path}`") + + def _run_clear(clear_type: _CLEAR_CHOICE) -> None: """Run the `clear` subcommand.""" cache_folder = resolve_cache_folder() if clear_type == "index" or clear_type == "all": - indexes = [] - for path in cache_folder.glob("*/index"): - if not _SHA_256_REGEX.match(path.parent.name): - continue - if PersistencePath.from_path(path).non_existing(): - continue - indexes.append(path) - - if not indexes: - print(f"No indexes found to clear in `{cache_folder}`") - else: - for path in indexes: - index_folder = path.parent - rmtree(index_folder) - print(f"Cleared index at `{index_folder}`") - + _clear_indexes(cache_folder) if clear_type == "savings" or clear_type == "all": - path = cache_folder / "savings.jsonl" - if not path.exists(): - print(f"No savings file found at `{path}`") - else: - path.unlink() - print(f"Cleared savings at `{path}`") + _clear_savings(cache_folder) + if clear_type == "orphans": + _clear_orphans(cache_folder) def _cli_main() -> None: @@ -186,7 +224,11 @@ def _cli_main() -> None: _add_content_args(search_p) clear_p = sub.add_parser("clear", help="Clear the index cache.") - clear_p.add_argument("type", choices=["all", "index", "savings"], help="Type of cache to clear.") + clear_p.add_argument( + "type", + choices=["all", "index", "savings", "orphans"], + help="Type of cache to clear. `orphans` removes indexes whose source path no longer exists.", + ) related_p = sub.add_parser("find-related", help="Find code similar to a specific location.") related_p.add_argument("file_path", help="File path as shown in search results.") diff --git a/tests/test_cli.py b/tests/test_cli.py index 539d71c1..e4245f60 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,3 +1,5 @@ +import hashlib +import json import sys import warnings from importlib.resources import files @@ -240,7 +242,7 @@ def test_agent_file_tools_are_bash_only() -> None: assert not any("mcp__" in t for t in tools) -def _make_valid_index_dir(cache_folder: Path, sha: str = "a" * 64) -> Path: +def _make_valid_index_dir(cache_folder: Path, sha: str = "a" * 64, metadata: str = "{}") -> Path: """Create a fake valid index directory with the expected structure.""" index_dir = cache_folder / sha / "index" index_dir.mkdir(parents=True) @@ -248,7 +250,7 @@ def _make_valid_index_dir(cache_folder: Path, sha: str = "a" * 64) -> Path: (index_dir / "chunks.json").write_text("[]") (index_dir / "bm25_index").write_text("") (index_dir / "semantic_index").write_text("") - (index_dir / "metadata.json").write_text("{}") + (index_dir / "metadata.json").write_text(metadata) return index_dir @@ -291,6 +293,64 @@ def test_run_clear_index( assert not (tmp_path / ("b" * 64)).exists() +@pytest.mark.parametrize( + "scenario", + ["orphan", "live", "mismatched_key", "no_root_path"], +) +def test_run_clear_orphans(scenario: str, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + """_run_clear('orphans') removes entries whose local root is gone, and keeps everything else.""" + cache_folder = tmp_path / "cache" + cache_folder.mkdir() + root = tmp_path / "repo" + root.mkdir() + sha = hashlib.sha256(str(root.resolve()).encode("utf-8")).hexdigest() + if scenario == "orphan": + _make_valid_index_dir(cache_folder, sha, metadata=json.dumps({"root_path": str(root)})) + root.rmdir() + elif scenario == "live": + _make_valid_index_dir(cache_folder, sha, metadata=json.dumps({"root_path": str(root)})) + elif scenario == "mismatched_key": + # A git-URL entry: the dir name hashes the URL, not the (missing) root_path + _make_valid_index_dir(cache_folder, "a" * 64, metadata=json.dumps({"root_path": str(root / "clone")})) + elif scenario == "no_root_path": + _make_valid_index_dir(cache_folder, "b" * 64) + + with patch("semble.cli.resolve_cache_folder", return_value=cache_folder): + _run_clear("orphans") + + out = capsys.readouterr().out + if scenario == "orphan": + assert str(root) in out + assert not (cache_folder / sha).exists() + else: + assert "No orphaned indexes found" in out + assert len(list(cache_folder.iterdir())) == 1 + + +def test_run_clear_orphans_skips_invalid_metadata(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + """Malformed cache entries are skipped without aborting the rest of the cleanup.""" + cache_folder = tmp_path / "cache" + cache_folder.mkdir() + root = tmp_path / "repo" + root.mkdir() + sha = hashlib.sha256(str(root.resolve()).encode("utf-8")).hexdigest() + _make_valid_index_dir(cache_folder, sha, metadata=json.dumps({"root_path": str(root)})) + root.rmdir() + _make_valid_index_dir(cache_folder, "f" * 64, metadata=json.dumps({"root_path": 123})) + _make_valid_index_dir(cache_folder, "0" * 64, metadata="not json") + _make_valid_index_dir(cache_folder, "1" * 64, metadata="[]") + (cache_folder / "not-a-sha" / "index").mkdir(parents=True) + + with patch("semble.cli.resolve_cache_folder", return_value=cache_folder): + _run_clear("orphans") + + out = capsys.readouterr().out + assert str(root) in out + assert not (cache_folder / sha).exists() + for kept in ("f" * 64, "0" * 64, "1" * 64, "not-a-sha"): + assert (cache_folder / kept).exists() + + @pytest.mark.parametrize( ("create_file", "expected"), [ @@ -348,6 +408,7 @@ def test_run_clear_all( ("index", True, False, ["Cleared index", "e" * 64]), ("savings", False, True, ["Cleared savings"]), ("all", True, True, ["Cleared index", "Cleared savings"]), + ("orphans", False, False, ["No orphaned indexes found"]), ], ) def test_cli_clear_command(