Build native modules for Mac

.github/workflows/ci.yml

@@ -56,7 +56,8 @@ jobs:
     env:
       CACHIX_CACHE_NAME: dimensionalos
     outputs:
-      needs-build: ${{ steps.decide.outputs.needs-build }}
+      needs-build-linux: ${{ steps.decide.outputs.linux }}
+      needs-build-macos: ${{ steps.decide.outputs.macos }}
       inputs-hash: ${{ steps.hash.outputs.inputs-hash }}
 
     steps:
@@ -65,24 +66,36 @@ jobs:
       - name: Compute native-module inputs hash
         id: hash
         run: python3 bin/build-native-modules --inputs-hash
-      - name: Probe publish marker
-        id: marker
+      - name: Probe Linux publish marker
+        id: marker-linux
         uses: actions/cache/restore@v6
         with:
           path: .cachix-marker
-          key: cachix-published-${{ runner.os }}-${{ runner.arch }}-${{ steps.hash.outputs.inputs-hash }}
+          key: cachix-published-Linux-X64-${{ steps.hash.outputs.inputs-hash }}
           lookup-only: true
-      - name: Decide whether a publish is needed
+      - name: Probe macOS publish marker
+        id: marker-macos
+        uses: actions/cache/restore@v6
+        with:
+          path: .cachix-marker
+          key: cachix-published-macOS-ARM64-${{ steps.hash.outputs.inputs-hash }}
+          lookup-only: true
+      - name: Decide which platforms need a publish
         id: decide
         env:
           FORCE: ${{ inputs.force-cachix-build }}
-          MARKER_HIT: ${{ steps.marker.outputs.cache-hit }}
+          LINUX_HIT: ${{ steps.marker-linux.outputs.cache-hit }}
+          MACOS_HIT: ${{ steps.marker-macos.outputs.cache-hit }}
         run: |
-          if [ "$FORCE" != "true" ] && [ "$MARKER_HIT" = "true" ]; then
-            echo "needs-build=false" >> "$GITHUB_OUTPUT"
-          else
-            echo "needs-build=true" >> "$GITHUB_OUTPUT"
-          fi
+          emit() {  # $1=output name, $2=this platform's marker hit
+            if [ "$FORCE" != "true" ] && [ "$2" = "true" ]; then
+              echo "$1=false" >> "$GITHUB_OUTPUT"
+            else
+              echo "$1=true" >> "$GITHUB_OUTPUT"
+            fi
+          }
+          emit linux "$LINUX_HIT"
+          emit macos "$MACOS_HIT"
 
   cachix-build:
     # ⚠️ NEVER RUN ON UNTRUSTED (fork) CODE ⚠️
@@ -101,7 +114,7 @@ jobs:
     if: |
       !cancelled() &&
       (github.event_name == 'push' || github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.full_name == github.repository) &&
-      needs.cachix-build-check.outputs.needs-build == 'true'
+      needs.cachix-build-check.outputs.needs-build-linux == 'true'
     needs: cachix-build-check
     timeout-minutes: 300
     environment: cachix
@@ -165,6 +178,71 @@ jobs:
           path: .cachix-marker
           key: cachix-published-${{ runner.os }}-${{ runner.arch }}-${{ needs.cachix-build-check.outputs.inputs-hash }}
 
+  cachix-build-macos:
+    # ⚠️ NEVER RUN ON UNTRUSTED (fork) CODE ⚠️ — holds CACHIX_AUTH_TOKEN, same
+    # as cachix-build; the `if:` guard below is the only thing keeping forks
+    # out. A separate job (not a matrix leg of cachix-build) for two reasons:
+    # its own job-level `if` must gate entry into the `cachix` environment on
+    # the macOS marker alone (matrix context isn't available in `if:`), and a
+    # macOS build failure must not fail cachix-build and block the Linux tests.
+    if: |
+      !cancelled() &&
+      (github.event_name == 'push' || github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.full_name == github.repository) &&
+      needs.cachix-build-check.outputs.needs-build-macos == 'true'
+    needs: cachix-build-check
+    timeout-minutes: 300
+    environment: cachix
+    runs-on: macos-14  # GitHub-hosted Apple silicon → runner.arch is ARM64
+    permissions:
+      contents: read  # For checkout
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+      - name: Install Nix (with Cachix substituter)
+        env:
+          INPUT_EXTRA_NIX_CONFIG: |
+            extra-substituters = https://dimensionalos.cachix.org
+            extra-trusted-public-keys = dimensionalos.cachix.org-1:20ynj6TjpoD3qTxkdNoeHtgs2G2pNvgAq1EQYLTHJXI=
+          INPUT_SET_AS_TRUSTED_USER: "true"
+        run: bash docker/ros/install-nix.sh
+      - name: Cache Nix fetcher + eval cache
+        # Key includes runner.os, so this is a macOS-only entry, distinct from
+        # the Linux fetcher cache.
+        uses: actions/cache@v6
+        with:
+          path: |
+            ~/.cache/nix
+            ~/.local/state/nix
+          key: nix-fetcher-build-${{ runner.os }}-${{ hashFiles('bin/build-native-modules', '**/flake.lock', '**/flake.nix') }}
+          restore-keys: nix-fetcher-build-${{ runner.os }}-
+      - name: Authenticate Cachix
+        uses: cachix/cachix-action@v17
+        with:
+          name: dimensionalos
+          authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
+      - name: Build native modules
+        env:
+          BUILD_WORKERS: "1"
+        run: python3 bin/build-native-modules
+      - name: Verify Cachix holds the built paths
+        env:
+          CACHIX_CACHE_NAME: dimensionalos
+        run: python3 bin/build-native-modules --verify-published
+      - name: Record the published inputs manifest
+        env:
+          CACHIX_CACHE_NAME: dimensionalos
+        run: |
+          mkdir -p .cachix-marker
+          python3 bin/build-native-modules --inputs-hash \
+            > .cachix-marker/inputs-hash.txt 2> .cachix-marker/manifest.txt
+          python3 bin/build-native-modules --record-links > .cachix-marker/links.txt
+      - name: Save publish marker
+        uses: actions/cache/save@v6
+        with:
+          path: .cachix-marker
+          key: cachix-published-${{ runner.os }}-${{ runner.arch }}-${{ needs.cachix-build-check.outputs.inputs-hash }}
+
   lint:
     timeout-minutes: 10
     runs-on: ubuntu-latest
@@ -646,7 +724,7 @@ jobs:
       (github.event_name == 'push' || github.event_name == 'merge_group' || github.event.pull_request.head.repo.full_name == github.repository) &&
       needs.cachix-build-check.result == 'success' &&
       contains(fromJSON('["success", "skipped"]'), needs.cachix-build.result)
-    needs: [compute-ros-pin, cachix-build-check, cachix-build]
+    needs: [compute-ros-pin, cachix-build-check, cachix-build, cachix-build-macos]
     env:
       OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
       ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
@@ -771,17 +849,25 @@ jobs:
           max-jobs = 0
           EOF
           echo "/nix/var/nix/profiles/default/bin" >> "$GITHUB_PATH"
+      - name: Put Nix on PATH (macOS)
+        if: matrix.os == 'macOS'
+        run: |
+          bin=/nix/var/nix/profiles/default/bin
+          if [ ! -x "$bin/nix-store" ]; then
+            echo "::error::Nix is not installed on this macOS runner host — install multi-user Nix and add the dimensionalos substituter to /etc/nix/nix.conf (see this step's comment)"
+            exit 1
+          fi
+          echo "$bin" >> "$GITHUB_PATH"
+          echo "NIX_CONFIG=experimental-features = nix-command flakes" >> "$GITHUB_ENV"
       - name: Restore publish marker
-        if: contains(matrix.markers, 'skipif_no_ros')
         id: native-marker
         uses: actions/cache/restore@v6
         with:
           path: .cachix-marker
           key: cachix-published-${{ runner.os }}-${{ runner.arch }}-${{ needs.cachix-build-check.outputs.inputs-hash }}
       - name: Provision native modules from Cachix
-        if: contains(matrix.markers, 'skipif_no_ros')
         # The marker records every result -> store path mapping, so the warm
-        # path is symlink recreation against the runner's persistent /nix plus
+        # path is symlink recreation against the persistent /nix plus
         # substitution of anything missing by exact path — no nix evaluation.
         # A missed marker (evicted) falls back to the full nix build.
         run: |
@@ -790,16 +876,18 @@ jobs:
           else
             python3 bin/build-native-modules
           fi
-          # Root the out paths (and thereby their closures) so the end-of-job
-          # GC keeps them warm: the workspace result links die in the next
-          # run's `git clean`, so they cannot serve as roots themselves.
-          # Replacing the directory unroots superseded paths for GC to reap.
-          sudo mkdir -p /nix/var/nix/gcroots/dimos-native
-          sudo find /nix/var/nix/gcroots/dimos-native -maxdepth 1 -type l -delete
-          python3 bin/build-native-modules --record-links | cut -d' ' -f2 | sort -u |
-            while read -r p; do
-              sudo ln -sfn "$p" "/nix/var/nix/gcroots/dimos-native/$(basename "$p")"
-            done
+          # Linux only: root the out paths so this job's end-of-GC keeps the
+          # closure warm in the CI-managed bind-mounted store (the workspace
+          # result links die in the next run's `git clean`, so they cannot
+          # serve as roots). The macOS host manages its own store and GC.
+          if [ "${{ matrix.os }}" = "Linux" ]; then
+            sudo mkdir -p /nix/var/nix/gcroots/dimos-native
+            sudo find /nix/var/nix/gcroots/dimos-native -maxdepth 1 -type l -delete
+            python3 bin/build-native-modules --record-links | cut -d' ' -f2 | sort -u |
+              while read -r p; do
+                sudo ln -sfn "$p" "/nix/var/nix/gcroots/dimos-native/$(basename "$p")"
+              done
+          fi
       - name: Run tests
         run: uv run pytest --cov=dimos/ --junitxml=junit.xml -m '(${{ matrix.markers }}) and not mujoco'
       - name: Re-run the failing tests with maximum verbosity

dimos/experimental/memory/test_rust_recorder_e2e.py

@@ -75,19 +75,21 @@ def stop(self) -> None:
 
 @pytest.fixture(scope="module")
 def rust_recorder_executable() -> Path:
-    subprocess.run(
-        [
-            "nix",
-            "--extra-experimental-features",
-            "nix-command flakes",
-            "build",
-            "-L",
-            ".#dimos-memory-recorder",
-            "--no-write-lock-file",
-        ],
-        cwd=_RUST_PACKAGE,
-        check=True,
-    )
+    # Build locally. Should already be built in CI.
+    if not _EXECUTABLE.is_file():
+        subprocess.run(
+            [
+                "nix",
+                "--extra-experimental-features",
+                "nix-command flakes",
+                "build",
+                "-L",
+                ".#dimos-memory-recorder",
+                "--no-write-lock-file",
+            ],
+            cwd=_RUST_PACKAGE,
+            check=True,
+        )
     assert _EXECUTABLE.is_file()
     return _EXECUTABLE