Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion bin/build-native-modules
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,10 @@ def _flake_refs(flake: Path) -> tuple[set[str], set[str]]:
f"{os.path.relpath(flake, REPO_ROOT)}: {what} reaches content the inputs hash"
" cannot follow; use a relative path literal or a locked flake input instead"
)
# `pkgs.lib.fileset.toSource { root = <dir>; fileset = <members>; }` anchors
# the enumerated members at `root`; the copied content is the members, not
# `root` itself (typically the repo root). Skip hashing the root itself.
uses_fileset = "fileset.toSource" in text
literals: set[str] = set()
path_inputs: set[str] = set()
for match in _FLAKE_REF.finditer(text):
Expand All @@ -323,6 +327,8 @@ def _flake_refs(flake: Path) -> tuple[set[str], set[str]]:
f" {match.group('path')!r} — the inputs hash can only follow static path"
" literals; reference a plain literal (a parent directory is fine) instead"
)
if uses_fileset and re.search(r"\broot\s*=\s*$", text[: match.start()]):
continue # fileset.toSource anchor, not an input
rel = _resolve_ref(flake, match.group("path"))
(path_inputs if match.group("scheme") == "path:" else literals).add(rel)
return literals, path_inputs
Expand Down Expand Up @@ -377,9 +383,10 @@ def build_manifest(modules: tuple[DiscoveredModule, ...]) -> str:
for module in modules:
paths |= _collect_input_paths(module)
# Drop paths nested under another (e.g. a patch file inside its build dir).
# "." (repo root) sorts first and subsumes everything.
top: list[str] = []
for path in sorted(paths):
if not any(path == kept or path.startswith(kept + "/") for kept in top):
if not any(kept == "." or path == kept or path.startswith(kept + "/") for kept in top):
top.append(path)
lines = [
f"salt {MARKER_SALT}",
Expand Down
37 changes: 36 additions & 1 deletion dimos/core/test_build_native_modules.py
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,38 @@ def test_ast_extraction_matches_runtime() -> None:
assert runtime_dir == (DIMOS_PROJECT_ROOT / module.build_dir).resolve()


def test_no_module_hashes_the_repo_root() -> None:
"""A collected input of "." puts the whole-repo tree SHA in the marker key,
so it changes on every commit and the marker never matches. A
fileset.toSource `root` anchor (usually the repo root) must be skipped, not
hashed — regression guard for the rust_recorder fileset flake."""
for module in _SCRIPT.discover():
assert "." not in _SCRIPT._collect_input_paths(module), (
f"{module.qualname}: input set includes the repo root — a fileset root anchor "
"is being hashed, which busts the publish marker on every commit"
)


def test_recorder_fileset_covers_every_workspace_member() -> None:
"""Cargo resolves the workspace from the root manifest, so the recorder's
fileset src must carry every [workspace] member — as static path literals,
because the publish gate can only hash literals. Deriving the list from
Cargo.toml at eval time (fromTOML) is invisible to the flake parser, which
then hashes the fileset root instead: the repo-root tree SHA busts the
publish marker on every commit."""
flake = DIMOS_PROJECT_ROOT / "dimos" / "experimental" / "memory" / "rust" / "flake.nix"
block = re.search(r"members\s*=\s*\[([^]]*)\]", (DIMOS_PROJECT_ROOT / "Cargo.toml").read_text())
assert block is not None, "no [workspace] members array in Cargo.toml"
members = re.findall(r'"([^"]+)"', block.group(1))
assert members, "no [workspace] members parsed from Cargo.toml"
literals, _path_inputs = _SCRIPT._flake_refs(flake)
for member in members:
assert any(member == lit or member.startswith(lit + "/") for lit in literals), (
f"workspace member {member!r} has no covering path literal in {flake} — "
"list it in the fileset.unions so the publish gate hashes it"
)


@pytest.mark.skipif(not _IN_GIT_CHECKOUT, reason="needs git HEAD for object hashes")
def test_manifest_is_deterministic() -> None:
modules = _SCRIPT.discover()
Expand Down Expand Up @@ -270,7 +302,8 @@ def test_flake_refs_resolve_and_are_covered() -> None:
flake = DIMOS_PROJECT_ROOT / rel / "flake.nix"
if not flake.is_file():
continue # plain source tree (e.g. native/cpp), nothing to sweep
for match in _RAW_REF.finditer(flake.read_text()):
raw = flake.read_text()
for match in _RAW_REF.finditer(raw):
if match.group("scheme") == "git+file:":
url = match.group("path").split("?", 1)[0]
lock = json.loads((flake.parent / "flake.lock").read_text())
Expand All @@ -286,6 +319,8 @@ def test_flake_refs_resolve_and_are_covered() -> None:
" derivation on every commit behind the publish gate's back"
)
continue
if "fileset.toSource" in raw and re.search(r"\broot\s*=\s*$", raw[: match.start()]):
continue # fileset anchor, deliberately not an input (see _flake_refs)
token = match.group("path").split("?", 1)[0]
target = os.path.relpath(os.path.normpath(flake.parent / token), DIMOS_PROJECT_ROOT)
if not (DIMOS_PROJECT_ROOT / target).exists():
Expand Down
27 changes: 15 additions & 12 deletions dimos/experimental/memory/rust/flake.nix
Original file line number Diff line number Diff line change
Expand Up @@ -17,22 +17,25 @@
"aarch64-darwin"
] (system: let
pkgs = nixpkgs.legacyPackages.${system};
workspaceRoot = ../../../..;
# resolve the members straight from the workspace
workspaceMembers =
(builtins.fromTOML (builtins.readFile (workspaceRoot + "/Cargo.toml"))).workspace.members;
dimos-memory-recorder = pkgs.rustPlatform.buildRustPackage {
pname = "dimos-memory-recorder";
version = "0.1.0";
src = pkgs.lib.fileset.toSource {
root = workspaceRoot;
fileset = pkgs.lib.fileset.unions (
[
(workspaceRoot + "/Cargo.lock")
(workspaceRoot + "/Cargo.toml")
]
++ map (member: workspaceRoot + "/${member}") workspaceMembers
);
root = ../../../..;
fileset = pkgs.lib.fileset.unions [
../../../../Cargo.lock
../../../../Cargo.toml
../../../../dimos/experimental/memory/rust
../../../../native/rust/dimos-module
../../../../native/rust/dimos-module-macros
../../../../dimos/mapping/ray_tracing/rust
../../../../dimos/mapping/ray_tracing/rust/py
../../../../dimos/navigation/nav_3d/mls_planner/rust
../../../../dimos/navigation/nav_3d/mls_planner/rust/py
../../../../dimos/hardware/sensors/lidar/livox/rust
../../../../dimos/hardware/sensors/lidar/virtual_mid360
../../../../examples/native-modules/rust
];
};

cargoLock = {
Expand Down
Loading