diff --git a/pyproject.toml b/pyproject.toml index bcfcf8bb2..d478d7819 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,9 +32,9 @@ dependencies = [ "bm25s>=0.2.0", "pathspec>=0.12", "tree-sitter>=0.25,<0.26", - "tree-sitter-language-pack>=1.0,<1.8.0,!=1.6.3", "orjson", "questionary>=2.0,<3.0", + "semble-grammars>=0.1.2", ] [project.optional-dependencies] @@ -146,3 +146,6 @@ exclude_also = ["if __name__ == .__main__.:", "@abstractmethod"] [tool.uv] exclude-newer = "1 week" + +[tool.uv.exclude-newer-package] +semble-grammars = false diff --git a/src/semble/chunking/chunking.py b/src/semble/chunking/chunking.py index e0f3f2f87..e9ddf71ca 100644 --- a/src/semble/chunking/chunking.py +++ b/src/semble/chunking/chunking.py @@ -1,6 +1,6 @@ import logging -from semble.chunking.core import chunk, chunk_lines, is_supported_language +from semble.chunking.core import chunk, chunk_lines from semble.types import Chunk logger = logging.getLogger(__name__) @@ -15,7 +15,7 @@ def chunk_source(source: str, file_path: str, language: str | None) -> list[Chun if not source.strip(): return [] chunk_boundaries = None - if language is not None and is_supported_language(language): + if language is not None: chunk_boundaries = chunk(source, language, _DESIRED_CHUNK_LENGTH_CHARS) # This is an if because the error state of the parser above # is a None. diff --git a/src/semble/chunking/core.py b/src/semble/chunking/core.py index 4b326f9d6..5a904bfbd 100644 --- a/src/semble/chunking/core.py +++ b/src/semble/chunking/core.py @@ -4,10 +4,8 @@ from functools import cache from logging import getLogger +from semble_grammars import LanguageNotFoundError, UnsupportedPlatformError, get_parser from tree_sitter import Node, Parser -from tree_sitter_language_pack import DownloadError, LanguageNotFoundError, SupportedLanguage, get_parser - -from semble.index.files import ALL_LANGUAGES logger = getLogger(__name__) @@ -15,11 +13,6 @@ _MIN_CHUNK_SIZE = 50 -def is_supported_language(language: str) -> bool: - """Check if the language is supported by tree-sitter.""" - return language in ALL_LANGUAGES - - @dataclass class ChunkBoundary: """The output of the internal chunking algorithm.""" @@ -29,14 +22,14 @@ class ChunkBoundary: @cache -def _cached_get_parser(language: SupportedLanguage) -> Parser | None: +def _cached_get_parser(language: str) -> Parser | None: """Gets a parser from tree_sitter.""" try: return get_parser(language) except LanguageNotFoundError: logger.warning("Language %s not found, falling back to line chunking", language) - except DownloadError: - logger.warning("Failed to download language %s, falling back to line chunking", language) + except UnsupportedPlatformError: + logger.warning("No bundled grammars for this platform, falling back to line chunking") except Exception: logger.error("Uncaught exception in _cached_get_parser", exc_info=True) return None diff --git a/src/semble/installer/config.py b/src/semble/installer/config.py index 3cb1cc8b8..5208c9e9d 100644 --- a/src/semble/installer/config.py +++ b/src/semble/installer/config.py @@ -1,11 +1,12 @@ from __future__ import annotations import json +from functools import cache from pathlib import Path -from typing import Literal, TypeVar, cast +from typing import Literal, TypeVar +from semble_grammars import get_parser from tree_sitter import Node, Parser -from tree_sitter_language_pack import SupportedLanguage, download, get_parser from semble.installer.agents import SEMBLE_END, SEMBLE_START, Action @@ -15,23 +16,14 @@ _CODEX_MCP_HEADER = "[mcp_servers.semble]" _CODEX_MCP_BLOCK = '[mcp_servers.semble]\ncommand = "uvx"\nargs = ["--from", "semble[mcp]", "semble"]\n' -_json5_parser_cache: Parser | None | bool = False # False = not yet attempted - +@cache def _json5_parser() -> Parser | None: - """Return a tree-sitter JSON5 parser, downloading the grammar if needed. - - "json5" ships in tree-sitter-language-pack but isn't in its typed language list, hence the cast. - """ - global _json5_parser_cache - if _json5_parser_cache is not False: - return _json5_parser_cache # type: ignore[return-value] + """Return a tree-sitter JSON5 parser, or None if unavailable.""" try: - download(["json5"]) - _json5_parser_cache = get_parser(cast(SupportedLanguage, "json5")) + return get_parser("json5") except Exception: - _json5_parser_cache = None - return _json5_parser_cache # type: ignore[return-value] + return None def _json5_object(text: str) -> JsonObjectResult: diff --git a/src/semble/version.py b/src/semble/version.py index 0a9d44d91..aa9fde078 100644 --- a/src/semble/version.py +++ b/src/semble/version.py @@ -1,2 +1,2 @@ -__version_triple__ = (0, 5, 1) +__version_triple__ = (0, 5, 2) __version__ = ".".join(map(str, __version_triple__)) diff --git a/tests/test_chunker.py b/tests/test_chunker.py index aec7ec929..135b3f1d4 100644 --- a/tests/test_chunker.py +++ b/tests/test_chunker.py @@ -2,7 +2,7 @@ from unittest.mock import patch import pytest -from tree_sitter_language_pack import DownloadError +from semble_grammars import UnsupportedPlatformError from semble.chunking.chunking import Chunk, chunk_lines, chunk_source from semble.chunking.core import ChunkBoundary, _cached_get_parser, chunk @@ -102,12 +102,11 @@ def test_get_parser(caplog: pytest.LogCaptureFixture) -> None: _cached_get_parser("hello") assert len(caplog.records) == 0 - with patch("semble.chunking.core.get_parser", side_effect=DownloadError): + with patch("semble.chunking.core.get_parser", side_effect=UnsupportedPlatformError): with caplog.at_level(logging.WARNING, logger="semble.chunking.core"): _cached_get_parser("Python") assert len(caplog.records) == 1 - assert "Failed to download" in caplog.records[0].message - assert "Python" in caplog.records[0].message + assert "No bundled grammars for this platform" in caplog.records[0].message caplog.clear() _cached_get_parser("Python") @@ -131,9 +130,9 @@ def test_chunks_is_none() -> None: assert chunks is None -def test_download_error() -> None: - """Test that chunk returns None when parser is not available.""" - with patch("semble.chunking.core.get_parser", side_effect=DownloadError): +def test_unsupported_platform_error() -> None: + """Test that chunk returns None when the current platform has no bundled grammars.""" + with patch("semble.chunking.core.get_parser", side_effect=UnsupportedPlatformError): chunks = chunk("x = 1", "python", 10) assert chunks is None diff --git a/tests/test_installer.py b/tests/test_installer.py index 1c89052f1..c01e26b65 100644 --- a/tests/test_installer.py +++ b/tests/test_installer.py @@ -17,6 +17,7 @@ ) from semble.installer.config import ( _CODEX_MCP_HEADER, + _json5_parser, merge_json_member, merge_toml_block, remove_json_member, @@ -167,12 +168,13 @@ def test_merge_mcp_writes_under_agent_key(tmp_path, agent_id, key): def test_mcp_skipped_when_grammar_unavailable(claude_agent, monkeypatch): - """When the JSON5 grammar cannot be downloaded, merge/remove return 'skipped'.""" + """When the JSON5 grammar is unavailable, merge/remove return 'skipped'.""" claude_agent.mcp.path.write_text('{ "mcpServers": {} }') - monkeypatch.setattr("semble.installer.config.download", lambda _: 1 / 0) - monkeypatch.setattr("semble.installer.config._json5_parser_cache", False) + monkeypatch.setattr("semble.installer.config.get_parser", lambda _: 1 / 0) + _json5_parser.cache_clear() assert merge_mcp(claude_agent).action == "skipped" assert remove_mcp(claude_agent).action == "skipped" + _json5_parser.cache_clear() def test_merge_mcp_reparse_guard(claude_agent, monkeypatch): diff --git a/uv.lock b/uv.lock index f04d43a3e..774da529c 100644 --- a/uv.lock +++ b/uv.lock @@ -12,6 +12,9 @@ resolution-markers = [ exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values. exclude-newer-span = "P1W" +[options.exclude-newer-package] +semble-grammars = false + [[package]] name = "annotated-doc" version = "0.0.4" @@ -3150,8 +3153,8 @@ dependencies = [ { name = "orjson" }, { name = "pathspec" }, { name = "questionary" }, + { name = "semble-grammars" }, { name = "tree-sitter" }, - { name = "tree-sitter-language-pack" }, { name = "vicinity" }, ] @@ -3196,14 +3199,30 @@ requires-dist = [ { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5.0" }, { name = "questionary", specifier = ">=2.0,<3.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.9.0" }, + { name = "semble-grammars", specifier = ">=0.1.2" }, { name = "sentence-transformers", marker = "extra == 'benchmark'", specifier = ">=3.0" }, { name = "tiktoken", marker = "extra == 'benchmark'", specifier = ">=0.7" }, { name = "tree-sitter", specifier = ">=0.25,<0.26" }, - { name = "tree-sitter-language-pack", specifier = ">=1.0,!=1.6.3,<1.8.0" }, { name = "vicinity", specifier = ">=0.4.4" }, ] provides-extras = ["mcp", "benchmark", "dev"] +[[package]] +name = "semble-grammars" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tree-sitter" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/bd/2c0491654f9ce320faa72e6e047e07b11e696d8d737dad2a68629a294831/semble_grammars-0.1.2-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:52fe698f1d2489e6ad3e6462c9322e77796769b203f74a294f801e5a8bd10f19", size = 6690645, upload-time = "2026-08-01T11:24:46.97Z" }, + { url = "https://files.pythonhosted.org/packages/da/42/85eca6563cf3794a68e6b0fd6c03d3ed39b1466e68fd14e0b25b70084c19/semble_grammars-0.1.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:772e7011ce5e2ff5d9347a2f027a447c713d3f0d62983dd38457f04aa2215c6c", size = 7258590, upload-time = "2026-08-01T11:24:48.88Z" }, + { url = "https://files.pythonhosted.org/packages/32/e2/bec01f956cd7aa52505a57954f3259febd00092715eac6f0ea1fff6058e0/semble_grammars-0.1.2-py3-none-manylinux2014_aarch64.whl", hash = "sha256:f24f63a04e569f2de28aaf39166c8dddd8e296aa0390ac816df6ad1bd814e3bf", size = 6868770, upload-time = "2026-08-01T11:24:50.944Z" }, + { url = "https://files.pythonhosted.org/packages/9d/d9/e93eb269ccd2494cf4b5ac2f201b5badb959800bf21a2ae700d6d7f6418a/semble_grammars-0.1.2-py3-none-manylinux2014_x86_64.whl", hash = "sha256:c040595ca4e117969a349807dce05c1a51dbb368e80de80b005539fd86ef9c07", size = 6906388, upload-time = "2026-08-01T11:24:53.005Z" }, + { url = "https://files.pythonhosted.org/packages/26/d0/3a783d2eda7e4f8f631ae1890cb87bdfaa6cb97efbc74135b66641b89dde/semble_grammars-0.1.2-py3-none-win_amd64.whl", hash = "sha256:078ae79ad15442a2922dae46620f778fa237b191ac06e9c29832f079915d4d39", size = 11078076, upload-time = "2026-08-01T11:24:55.007Z" }, + { url = "https://files.pythonhosted.org/packages/04/20/96078011af0d3b0e5b1c1f4b8937ba6f89f7dd5edef71902e9949845e513/semble_grammars-0.1.2-py3-none-win_arm64.whl", hash = "sha256:6bb406d0fc9594b9bfd3cad82fb538748109d6c72066c0e76ea102c64f12c4d9", size = 10363135, upload-time = "2026-08-01T11:24:57.472Z" }, +] + [[package]] name = "sentence-transformers" version = "5.5.0" @@ -3603,20 +3622,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a6/6e/e64621037357acb83d912276ffd30a859ef117f9c680f2e3cb955f47c680/tree_sitter-0.25.2-cp314-cp314-win_arm64.whl", hash = "sha256:b8d4429954a3beb3e844e2872610d2a4800ba4eb42bb1990c6a4b1949b18459f", size = 117470, upload-time = "2025-09-25T17:37:58.431Z" }, ] -[[package]] -name = "tree-sitter-language-pack" -version = "1.6.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "tree-sitter" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/09/bd/ac34ab0ee92b2d27802754c575965e921490ce11b5357bf89f74a78e8309/tree_sitter_language_pack-1.6.2-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:f5998cfee5735a8e7e691f577062ff7eb3a7ea405ae5654c9cecaa4a1e6c81b0", size = 2241997, upload-time = "2026-04-18T07:04:36.042Z" }, - { url = "https://files.pythonhosted.org/packages/a1/e0/b997b8c3e0886288a47890e6313c3a7e74ea8192e2d141b3eab64d59a276/tree_sitter_language_pack-1.6.2-cp310-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:8ce814ede4e295f3419ba179b523889c52cc3a998ac085356a470e776596c026", size = 2419565, upload-time = "2026-04-18T07:04:37.67Z" }, - { url = "https://files.pythonhosted.org/packages/fa/a4/629e6983a93fbb52dc50af495ec0431565c6477eea4680d4298238e9831e/tree_sitter_language_pack-1.6.2-cp310-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:2305df7835c1cb3d34b71450b79d135878bc25ea5d02d9984cee864607a4ad60", size = 2555465, upload-time = "2026-04-18T07:04:39.57Z" }, - { url = "https://files.pythonhosted.org/packages/b5/9c/0f486ca7344f6f3345441e8516b464214c7c5a0f3775d11fda1368901c38/tree_sitter_language_pack-1.6.2-cp310-abi3-win_amd64.whl", hash = "sha256:08351222b43c3a73665571eaa440366add2093a2492bb35f032fb7a31945e720", size = 2351156, upload-time = "2026-04-18T07:04:41.377Z" }, -] - [[package]] name = "triton" version = "3.6.0"