Improve multimodal reasoning and runtime resilience

Makefile

@@ -730,12 +730,16 @@ test-molmo2: test-molmo2-contract test-molmo2-processor molmo2-model-compile sam
 test-molmo2-real: samosa-molmo2
 	@test -n "$(MOLMO2_MODEL_DIR)" || { echo "set MOLMO2_MODEL_DIR to the pinned molmo2-4b-mlx-q4-v1 package" >&2; exit 2; }
 	@test -n "$(MOLMO2_SAMPLE_IMAGE)" || { echo "set MOLMO2_SAMPLE_IMAGE to a reviewed image fixture" >&2; exit 2; }
+	@test -n "$(MOLMO2_SAMPLE_IMAGE_2)" || { echo "set MOLMO2_SAMPLE_IMAGE_2 to a second reviewed image fixture" >&2; exit 2; }
 	@test -n "$(MOLMO2_SAMPLE_VIDEO)" || { echo "set MOLMO2_SAMPLE_VIDEO to a reviewed short video fixture" >&2; exit 2; }
 	MAX_FOOTPRINT_MB=$${MAX_FOOTPRINT_MB:-3750} MAX_SWAP_DELTA_MB=0 \
 	METAL_PATH=$(MLX_BUILD_DIR)/mlx/backend/metal/kernels \
 	sh tools/run_memory_guarded.sh $(BUILD_DIR)/samosa-molmo2 --self-test image "$(MOLMO2_MODEL_DIR)" "$(MOLMO2_SAMPLE_IMAGE)"
 	MAX_FOOTPRINT_MB=$${MAX_FOOTPRINT_MB:-3750} MAX_SWAP_DELTA_MB=0 \
 	METAL_PATH=$(MLX_BUILD_DIR)/mlx/backend/metal/kernels \
+	sh tools/run_memory_guarded.sh $(BUILD_DIR)/samosa-molmo2 --self-test images "$(MOLMO2_MODEL_DIR)" "$(MOLMO2_SAMPLE_IMAGE)" "$(MOLMO2_SAMPLE_IMAGE_2)"
+	MAX_FOOTPRINT_MB=$${MAX_FOOTPRINT_MB:-3750} MAX_SWAP_DELTA_MB=0 \
+	METAL_PATH=$(MLX_BUILD_DIR)/mlx/backend/metal/kernels \
 	sh tools/run_memory_guarded.sh $(BUILD_DIR)/samosa-molmo2 --self-test video "$(MOLMO2_MODEL_DIR)" "$(MOLMO2_SAMPLE_VIDEO)"
 
 # Release qualification gate. Unlike test-visionpsy, this target refuses to

README.md

@@ -44,10 +44,12 @@ Only one primary chat model is loaded in memory at a time. When the optional
 Molmo2 package is installed it handles admitted visual image and video work;
 VisionPsy-Nano 460M is the standard-image fallback when Molmo is absent. On
 constrained Macs, Samosa stops the primary model, runs one bounded Molmo
-operation, and unloads it. If a conversation starts with one image or video,
-Molmo's own text and point coordinates are returned directly while the selected
-text model restarts for later turns. Later/composite visual workflows may use a
-grounded, greedy, non-thinking synthesis pass. Molmo2 is a locally built,
+operation, and unloads it. Under any selected text model, Molmo's grounded
+observation is handed back to that selected model for one bounded,
+non-thinking final response. Two attached images are supplied to
+Molmo together as labelled native visual inputs, rather than captioned in
+isolation. Selecting Molmo itself returns Molmo's own text and point
+coordinates directly. Molmo2 is a locally built,
 hash-verified package: the app never
 downloads its 19.4 GB FP32 source checkpoint. Samosa contains zero runtime
 Python dependencies. The real 16 GiB-machine conversion and zero-swap

assets/app.html

@@ -4978,10 +4978,13 @@ <h2 id="deleteChatsTitle">Delete conversation?</h2>
         }
         if (choice?.finish_reason) {
           const meta=event.samosa||{};
+          const finishReason=String(choice.finish_reason);
           const tokensPerSecond=meta.tokens_per_second||event.timings?.predicted_per_second;
           els.speed.textContent=tokensPerSecond ? `${tokensPerSecond.toFixed(2)} tok/s` : "—";
           els.rss.textContent=meta.rss_gb ? `${meta.rss_gb.toFixed(2)} GB` : "—";
-          els.closure.textContent=(meta.thinking_closure||choice.finish_reason).replaceAll("_"," ");
+          els.closure.textContent=(finishReason === "stop" ? (meta.thinking_closure||finishReason) : finishReason).replaceAll("_"," ");
+          if (finishReason === "length" && !assistant.error)
+            assistant.error="The model reached the response limit before finishing. The partial answer was kept; ask it to continue or retry the turn.";
           if (meta.compacted) els.closure.textContent=`auto-compacted ${Number(meta.compacted_from_tokens).toLocaleString()} → ${Number(meta.compacted_to_tokens).toLocaleString()} · ${els.closure.textContent}`;
           if (meta.session_saved===false) assistant.error="The answer completed, but its continuation snapshot could not be saved.";
         }

dist/samosa

@@ -85,6 +85,7 @@ else
 fi
 LAUNCH_DOMAIN="gui/$(id -u)"
 CURL="${SAMOSA_CURL:-curl}"
+LAUNCHCTL="${SAMOSA_LAUNCHCTL:-launchctl}"
 if [ "$(uname -s)" = "Darwin" ]; then
   OPEN="${SAMOSA_OPEN:-open}"
 else
@@ -394,7 +395,7 @@ serve_foreground() {
 
 launchd_available() {
   [ "${SAMOSA_DISABLE_LAUNCHD:-0}" != 1 ] &&
-    [ "$(uname -s)" = "Darwin" ] && command -v launchctl >/dev/null 2>&1
+    [ "$(uname -s)" = "Darwin" ] && command -v "$LAUNCHCTL" >/dev/null 2>&1
 }
 
 xml_escape() {
@@ -463,12 +464,19 @@ EOF
 
 start_server_launchd() {
   write_launchd_plist
-  launchctl bootout "$LAUNCH_DOMAIN/$LAUNCH_LABEL" >/dev/null 2>&1 || true
-  launchctl bootstrap "$LAUNCH_DOMAIN" "$LAUNCHD_PLIST" >/dev/null 2>&1 || {
-    echo "Samosa could not register its macOS background service; see $SERVER_LOG" >&2
-    return 1
-  }
-  return 0
+  "$LAUNCHCTL" bootout "$LAUNCH_DOMAIN/$LAUNCH_LABEL" >/dev/null 2>&1 || true
+  # launchd can acknowledge bootout before the old job has fully left its
+  # domain. An immediate bootstrap then fails once even though the plist is
+  # valid. Retry that bounded teardown race instead of reporting a false app
+  # startup failure to the user.
+  i=0
+  while [ "$i" -lt 20 ]; do
+    "$LAUNCHCTL" bootstrap "$LAUNCH_DOMAIN" "$LAUNCHD_PLIST" >/dev/null 2>&1 && return 0
+    sleep 0.1
+    i=$((i + 1))
+  done
+  echo "Samosa could not register its macOS background service after retrying; see $SERVER_LOG" >&2
+  return 1
 }
 
 start_server_detached() {
@@ -568,7 +576,7 @@ stop_server() {
   # Boot out only after giving the gateway an authenticated graceful-stop
   # request. Doing this first hid the shutdown cause as a generic SIGTERM.
   if launchd_available; then
-    launchctl bootout "$LAUNCH_DOMAIN/$LAUNCH_LABEL" >/dev/null 2>&1 || true
+    "$LAUNCHCTL" bootout "$LAUNCH_DOMAIN/$LAUNCH_LABEL" >/dev/null 2>&1 || true
   fi
   if server_running; then
     i=0

docs/INSTALL.md

@@ -114,6 +114,7 @@ Models** after the package is present. The real-release gate is:
 ```sh
 MOLMO2_MODEL_DIR="$HOME/.samosa/models/molmo2-4b-mlx-q4-v1" \
 MOLMO2_SAMPLE_IMAGE=/absolute/path/to/reviewed-image.png \
+MOLMO2_SAMPLE_IMAGE_2=/absolute/path/to/second-reviewed-image.png \
 MOLMO2_SAMPLE_VIDEO=/absolute/path/to/reviewed-video.mp4 \
 make test-molmo2-real
 ```

docs/MODELS_AND_INTERNET.md

@@ -96,7 +96,9 @@ After a valid package is installed, inference is offline. Text-only work never
 starts Molmo2. Every admitted visual image task, plus video, multi-image
 comparison, temporal tracking, and localization, prefers Molmo2 and acquires
 the global specialist lease. VisionPsy remains the standard-image fallback on
-installations without Molmo2. On constrained Macs the primary backend is stopped first.
+installations without Molmo2. A two-image request is sent as one
+labelled native Molmo2 multi-image prompt; it is not reduced to separate image
+captions. On constrained Macs the primary backend is stopped first.
 The gateway decodes bounded timestamp windows sequentially, terminates Molmo2,
 then restarts the primary model for a greedy, non-thinking grounded evidence synthesis.
 That synthesis is explicitly told that the local specialist inspected the

docs/MOLMO2_4B_NATIVE_PLAN.md

@@ -76,6 +76,7 @@ than interpreted at runtime. The important fixed values are:
 | Image input | 378 x 378, patch size 14, 729 learned positions |
 | Connector | hidden 1152, intermediate 9728, 16 heads, output width 2560; selected vision layers -3 and -9 |
 | Image processing | mean/std 0.5, bilinear resize, global plus overlap crops, maximum 8 crops, 2 x 2 pooling |
+| Multi-image processing | two labelled native image inputs in one decoder sequence; shared crop budget and sequential per-image vision encoding |
 | Video processing | timestamped 378 x 378 frames, 3 x 3 pooling, task-bounded sampling up to 2 fps |
 
 Molmo2 prefill is not a stock causal Qwen prefill: visual query and visual key
@@ -140,6 +141,15 @@ detailed visual reasoning. A request that requires Molmo2 returns an actionable
 `molmo2_model_required` or `molmo2_resource_pressure` result; it is never
 silently answered by a provider lacking the required capability.
 
+For two image attachments, the helper receives one bounded `images`
+command containing every validated private path. Its prompt and visual tokens
+follow the upstream `Image 1`, `Image 2` chat-template convention.
+Each image is vision-encoded sequentially to prevent crop memory from scaling
+with the batch, while the projected visual features are concatenated before
+the language decoder so cross-image reasoning remains genuinely joint. Three
+or more images fail explicitly rather than being silently split into unrelated
+observations.
+
 ### Native helper and IPC
 
 `samosa-molmo2` is a C++20 MLX/Metal executable. It reads only local, verified
@@ -418,9 +428,8 @@ All gates are mandatory:
 1. Text-only and OCR-only turns do not start Molmo2. Visual image turns prefer
    Molmo2 when its verified package is present; VisionPsy is the fallback.
 2. Molmo2 starts only after successful resource admission. It is gone before
-   any later/composite primary-model synthesis. For selected Molmo and for a
-   conversation's first single-visual turn, there is no primary synthesis and
-   Molmo is gone before the response completes.
+   primary-model synthesis. When Molmo itself is selected there is no primary
+   synthesis, and Molmo is gone before the response completes.
 3. The shipped runtime contains no Python invocation or dependency.
 4. Native image preprocessing, video timestamps, special-token placement, and
    visual attention masks match the pinned contract, including no column tokens
@@ -446,10 +455,10 @@ All gates are mandatory:
     responses contain Molmo's own output, text-only turns fail with an explicit
     visual-required error, and every completed/cancelled turn returns to zero
     Molmo child processes.
-14. A first image/video turn under any selected text model returns Molmo's own
-    output and exact coordinate markup without a planner or synthesis model
-    pass; the selected text model remains the conversation binding for later
-    text turns.
+14. An image/video turn under any selected text model uses Molmo's grounded
+   output as evidence for that selected model's bounded synthesis pass. A
+   multi-image turn sends all original pixels in one Molmo inference and
+   preserves exact coordinate markup through the handoff.
 
 If the Q4 artifact misses either the quality or machine-safety gate, the catalog
 entry remains unavailable. The accepted remedy is further native quantization,

docs/USAGE.md

@@ -54,11 +54,13 @@ image task as well as video, multi-image comparison, temporal tracking,
 localization/pointing, and complex spatial reasoning. OCR-only work remains on
 the native reader. Molmo2 remains an auxiliary provider: the gateway admits one
 specialist globally, and on a Mac with 18 GiB RAM or less it stops the primary
-chat backend before loading Molmo2. When a conversation starts with one image
-or video, Samosa returns Molmo's own text and point markup directly, unloads it,
-and starts restoring the selected text model without waiting on that restart.
-Later/composite visual turns can still use a greedy, non-thinking grounded
-synthesis pass when another model is actually needed.
+chat backend before loading Molmo2. For two images, Samosa labels and
+passes every original image to a single Molmo inference; the vision encoder is
+scheduled per image to preserve the qualified memory ceiling, then the decoder
+reasons across the combined visual feature sequence. Samosa unloads Molmo,
+restores the selected text model, and uses a greedy, non-thinking grounded
+synthesis pass for the final response. Selecting Molmo itself returns its
+visual response directly.
 
 VisionPsy is available in the first release on macOS Apple Silicon. A text-only
 chat LLM can still use it: `/healthz` distinguishes native chat-model image

src/molmo2/molmo2_model.cpp

@@ -328,6 +328,56 @@ struct Model::Impl {
             hidden = text_layers[layer].forward(hidden, mask, &(*caches)[layer]);
         return lm_head(rms_norm(hidden, final_norm));
     }
+
+    array encode_visual(const VisualInput& input) const {
+        if (input.segment_crop_counts.size() <= 1)
+            return vision.forward(input);
+        if (input.segment_crop_counts.size() !=
+            input.segment_patch_token_counts.size())
+            throw std::runtime_error("Molmo2 visual segment metadata mismatch");
+        constexpr std::size_t values_per_crop = 729 * 588;
+        std::vector<array> encoded;
+        encoded.reserve(input.segment_crop_counts.size());
+        int crop_offset = 0, token_offset = 0;
+        for (std::size_t segment = 0;
+             segment < input.segment_crop_counts.size(); ++segment) {
+            const int crops = input.segment_crop_counts[segment];
+            const int patch_tokens = input.segment_patch_token_counts[segment];
+            if (crops <= 0 || patch_tokens <= 0 ||
+                crop_offset + crops > input.crop_count ||
+                token_offset + patch_tokens > input.patch_token_count)
+                throw std::runtime_error("Molmo2 visual segment exceeds input bounds");
+            VisualInput current;
+            current.crop_count = crops;
+            current.patch_token_count = patch_tokens;
+            current.pooling_width = input.pooling_width;
+            const std::size_t patch_begin =
+                static_cast<std::size_t>(crop_offset) * values_per_crop;
+            const std::size_t patch_end = patch_begin +
+                static_cast<std::size_t>(crops) * values_per_crop;
+            current.patches.assign(input.patches.begin() + patch_begin,
+                                   input.patches.begin() + patch_end);
+            const std::size_t pool_begin =
+                static_cast<std::size_t>(token_offset) * input.pooling_width;
+            const std::size_t pool_end = pool_begin +
+                static_cast<std::size_t>(patch_tokens) * input.pooling_width;
+            current.pooling.reserve(pool_end - pool_begin);
+            const std::int32_t index_offset = crop_offset * 729;
+            for (std::size_t index = pool_begin; index < pool_end; ++index)
+                current.pooling.push_back(input.pooling[index] < 0
+                    ? -1 : input.pooling[index] - index_offset);
+            auto features = vision.forward(current);
+            eval(features);
+            encoded.push_back(std::move(features));
+            mlx::core::clear_cache();
+            crop_offset += crops;
+            token_offset += patch_tokens;
+        }
+        if (crop_offset != input.crop_count ||
+            token_offset != input.patch_token_count)
+            throw std::runtime_error("Molmo2 visual segments do not cover the input");
+        return concatenate(encoded, 0);
+    }
 };
 
 Model::Model() : impl_(std::make_unique<Impl>()) {}
@@ -422,7 +472,7 @@ GenerateResult Model::generate(const std::string& question, const VisualInput& v
 
         auto ids = array(tokens.data(), {1, static_cast<int>(tokens.size())}, int32);
         auto hidden = impl_->embed(ids);
-        auto visual_features = impl_->vision.forward(visual);
+        auto visual_features = impl_->encode_visual(visual);
         std::vector<int> patch_positions;
         for (std::size_t i = 0; i < tokens.size(); ++i)
             if (tokens[i] == 151938) patch_positions.push_back(static_cast<int>(i));