PR: Refactor setup and expose embedder backend in first-run

.env.example

@@ -1,36 +1,71 @@
-# Memayu configuration — copy to `.env` and fill in.
-# All settings are optional except the provider keys you want to use.
-# `make dev` loads this file automatically if present.
+# ==============================================================
+#  Memayu configuration
+#  Copy this file to `.env` and fill in your values.
+#  Everything is optional except the provider keys you plan to use.
+#  `make dev` loads `.env` automatically if it is present.
+# ==============================================================
 
-# --- Storage ---
+# ---------------------------------------------------------------
+#  Server
+# ---------------------------------------------------------------
+MEMAYU_BIND_ADDR=0.0.0.0
+MEMAYU_PORT=18080
+
+
+# ---------------------------------------------------------------
+#  Storage
+# ---------------------------------------------------------------
 # Backend: libsql (embedded, default) or postgres.
 MEMAYU_STORAGE_BACKEND=libsql
+
 # libsql database file (default: ./memayu.db)
 MEMAYU_LIBSQL_PATH=./memayu.db
+
 # Required when MEMAYU_STORAGE_BACKEND=postgres
 # MEMAYU_DATABASE_URL=postgres://user:pass@host:5432/memayu
 
-# --- LLM provider (extraction) — OpenAI-compatible ---
-# e.g. DeepSeek, Gemini, OpenRouter, Ollama Cloud
-MEMAYU_LLM_BASE_URL=https://api.deepseek.com/v1
-MEMAYU_LLM_API_KEY=
-MEMAYU_LLM_MODEL=deepseek-chat
-
-# --- Embedding provider — OpenAI-compatible ---
-# e.g. OpenAI, Gemini, Ollama Cloud
-MEMAYU_EMBEDDER_BASE_URL=https://api.openai.com/v1
-MEMAYU_EMBEDDER_API_KEY=
-MEMAYU_EMBEDDER_MODEL=text-embedding-3-small
-
-# --- Behavior ---
-# Expected embedding dimension (auto-detected on startup if unset)
-# MEMAYU_EMBEDDING_DIM=768
+
+# ---------------------------------------------------------------
+#  Extraction mode
+# ---------------------------------------------------------------
+# raw (default): store text verbatim, skip the LLM; dedupe threshold 0.98.
+# llm: extract + merge memory via the LLM provider below.
+MEMAYU_EXTRACTION_MODE=raw
+
+
+# ---------------------------------------------------------------
+#  LLM provider — OpenAI-compatible
+#  Only needed when MEMAYU_EXTRACTION_MODE=llm.
+# ---------------------------------------------------------------
+# MEMAYU_LLM_BASE_URL=https://api.deepseek.com/v1
+# MEMAYU_LLM_API_KEY=sk-
+# MEMAYU_LLM_MODEL=deepseek-chat
+
+
+# ---------------------------------------------------------------
+#  Embedding provider
+# ---------------------------------------------------------------
+# Choose ONE backend:
+#
+#   MEMAYU_EMBEDDER_BACKEND=local   (default) — runs a local embedder model
+#   MEMAYU_EMBEDDER_BACKEND=remote  — OpenAI-compatible / BYOK.
+MEMAYU_EMBEDDER_BACKEND=local
+
+# --- Local backend (on-device) ---
+# Default embedder. No API key or network calls needed.
+MEMAYU_EMBEDDER_MODEL=sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
+#
+# MEMAYU_EMBEDDER_DIM=768
+
+# --- Remote backend (BYOK) ---
+# MEMAYU_EMBEDDER_BACKEND=remote
+# MEMAYU_EMBEDDER_BASE_URL=https://api.openai.com/v1
+# MEMAYU_EMBEDDER_API_KEY=sk-
+# MEMAYU_EMBEDDER_MODEL=text-embedding-3-small
+
+
+# ---------------------------------------------------------------
+#  Behavior
+# ---------------------------------------------------------------
 # ADD vs UPDATE candidate similarity threshold (default 0.85)
 # MEMAYU_SIMILARITY_THRESHOLD=0.85
-# Extraction mode: llm (default, extract + merge via LLM) or raw
-# (store verbatim, skip LLM; dedupe threshold 0.98)
-# MEMAYU_EXTRACTION_MODE=llm
-
-# --- Server ---
-MEMAYU_BIND_ADDR=0.0.0.0
-MEMAYU_PORT=18080

Cargo.toml

@@ -10,6 +10,7 @@ members = [
     "crates/memayu-api",
     "crates/memayu-web",
     "crates/memayu-mcp",
+    "crates/memayu-setup",
     "bin/memayu",
 ]
 

README.md

@@ -217,7 +217,7 @@ The embedder can run fully on-device instead of calling a remote API. Set the em
 
 ```toml
 [embedder]
-backend = "local"               # "local" (on-device Candle) or "http" (BYOK, default)
+backend = "local"               # "local" (on-device Candle) or "remote" (BYOK, default)
 model   = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"
 ```
 

bin/memayu/Cargo.toml

@@ -11,6 +11,7 @@ memayu-storage-libsql = { path = "../../crates/memayu-storage-libsql" }
 memayu-storage-postgres = { path = "../../crates/memayu-storage-postgres" }
 memayu-llm-client = { path = "../../crates/memayu-llm-client" }
 memayu-identity = { path = "../../crates/memayu-identity" }
+memayu-setup = { path = "../../crates/memayu-setup" }
 
 # Optional frontend plugins
 memayu-api = { path = "../../crates/memayu-api", optional = true }

bin/memayu/src/main.rs

@@ -1,17 +1,16 @@
 mod cli;
 mod doctor;
 mod service;
-mod setup_flow;
 #[cfg(feature = "tui")]
 mod tui;
 #[cfg(feature = "tui")]
 mod tui_setup;
 mod wizard;
 
-#[cfg(feature = "web")]
-use memayu_config::StorageBackend;
 use memayu_config::{config_path, Config};
 #[cfg(feature = "web")]
+use memayu_config::{load_infrastructure, StorageBackend};
+#[cfg(feature = "web")]
 use memayu_core::MemoryService;
 #[cfg(feature = "web")]
 use std::io::IsTerminal;
@@ -67,8 +66,14 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         }
         #[cfg(feature = "web")]
         "serve" => {
-            let config = Config::load()?;
-            cmd_serve(config).await?;
+            // On a fresh, unconfigured install there is no fully-valid config
+            // yet. Rather than fail hard, boot a minimal setup-only server
+            // (Category A only) so first-run setup can be completed in the
+            // browser (#55).
+            match Config::load() {
+                Ok(config) => cmd_serve(config).await?,
+                Err(_) => cmd_serve_setup().await?,
+            }
         }
         #[cfg(feature = "mcp")]
         "mcp" => {
@@ -265,9 +270,23 @@ async fn cmd_serve(config: Config) -> Result<(), Box<dyn std::error::Error>> {
         std::process::exit(1);
     }
 
-    let service = build_web_service(&config).await?;
-
     let api_db = memayu_api::open_db(&config.storage).await?;
+
+    // #55: enforce config/DB precedence at boot. When the provider table is
+    // empty and the on-disk config is valid, seed it (so DB becomes
+    // authoritative); when empty, invalid, and no admin exists yet, this is an
+    // unconfigured first boot and we fail fast with an actionable error naming
+    // `memayu setup`.
+    memayu_api::ensure_provider_config(&api_db, &config)
+        .await
+        .map_err(|e| -> Box<dyn std::error::Error> { e.into() })?;
+
+    // DB-authoritative extraction mode, falling back to the config value on
+    // pre-seeding installs.
+    let extraction_mode = memayu_api::load_extraction_mode(&api_db, config.extraction_mode).await?;
+
+    let service = build_web_service(&config, extraction_mode).await?;
+
     let registry =
         memayu_api::load_registry(&api_db, config.llm.clone(), config.embedder.clone()).await?;
 
@@ -331,30 +350,39 @@ async fn cmd_mcp(config: Config) -> Result<(), Box<dyn std::error::Error>> {
 // ── shared service builders ──
 
 /// Build the dashboard service via the API registry, which powers runtime
-/// provider reconfiguration in the web UI.
+/// provider reconfiguration in the web UI. `extraction_mode` is the resolved,
+/// DB-authoritative value (falling back to config when nothing is stored yet).
 #[cfg(feature = "web")]
 async fn build_web_service(
     config: &Config,
+    extraction_mode: memayu_core::ExtractionMode,
 ) -> Result<Arc<MemoryService>, Box<dyn std::error::Error>> {
-    use memayu_core::{EmbedderProvider, StorageProvider};
+    use memayu_core::StorageProvider;
 
     let api_db = memayu_api::open_db(&config.storage).await?;
     let registry =
         memayu_api::load_registry(&api_db, config.llm.clone(), config.embedder.clone()).await?;
 
     let embedder = memayu_api::EmbedderConfigProvider::new(registry.clone());
-    let detected_dim = match config.dimension {
-        Some(d) => d,
-        None => embedder.embed("dimension probe").await?.len(),
-    };
-    println!(
-        "[memayu] embedder dimension = {detected_dim} (from {} {})",
-        config.embedder.base_url, config.embedder.model
-    );
+    // Resolve the embedding dimension from config or the existing schema — never
+    // by probing the embedder. The probe happens only in `memayu setup` (#55).
+    let resolved_dim = service::resolve_storage_dimension(config).await?;
+    // A local backend has no base_url; printing one (even a stale leftover)
+    // is confusing, so it's omitted from the message for local backends.
+    match config.embedder.backend {
+        memayu_config::EmbedderBackend::Local => println!(
+            "[memayu] embedder dimension = {resolved_dim} (local backend, {})",
+            config.embedder.model
+        ),
+        memayu_config::EmbedderBackend::Remote => println!(
+            "[memayu] embedder dimension = {resolved_dim} (from {} {})",
+            config.embedder.base_url, config.embedder.model
+        ),
+    }
 
     let storage: Arc<dyn StorageProvider> = match config.storage.backend {
         StorageBackend::Libsql => Arc::new(
-            memayu_storage_libsql::LibsqlProvider::open(&config.storage.libsql_path, detected_dim)
+            memayu_storage_libsql::LibsqlProvider::open(&config.storage.libsql_path, resolved_dim)
                 .await?,
         ),
         StorageBackend::Postgres => Arc::new(
@@ -364,7 +392,7 @@ async fn build_web_service(
                     .database_url
                     .as_deref()
                     .ok_or("missing postgres url")?,
-                detected_dim,
+                resolved_dim,
             )
             .await?,
         ),
@@ -373,6 +401,26 @@ async fn build_web_service(
     let llm = memayu_api::LlmConfigProvider::new(registry.clone());
     Ok(Arc::new(
         MemoryService::new(storage, Arc::new(embedder), Arc::new(llm))
-            .with_extraction_mode(config.extraction_mode),
+            .with_extraction_mode(extraction_mode),
     ))
 }
+
+/// Boot a minimal, setup-only web server (#55). Used when the process starts
+/// with a valid infrastructure slice but no fully-valid config — a fresh,
+/// unconfigured install. Only `/setup` (and static assets) are served so the
+/// user can complete first-run setup in the browser; the full dashboard and API
+/// are reachable only after setup writes a config and the server is restarted.
+#[cfg(feature = "web")]
+async fn cmd_serve_setup() -> Result<(), Box<dyn std::error::Error>> {
+    let infra = load_infrastructure()?;
+    let api_db = memayu_api::open_db(&infra.storage).await?;
+    let app = memayu_web::build_setup_router(api_db);
+
+    let addr = format!("{}:{}", infra.bind_addr, infra.port);
+    let listener = tokio::net::TcpListener::bind(&addr).await?;
+    println!("[memayu] setup-only mode (no valid config): listening on {addr}");
+    println!("[memayu] complete first-run setup at http://{addr}/setup");
+    println!("[memayu] restart the server after setup to start the full dashboard");
+    axum::serve(listener, app).await?;
+    Ok(())
+}

bin/memayu/src/service.rs

@@ -8,16 +8,44 @@ use memayu_core::{MemoryService, StorageProvider};
 use memayu_llm_client::{build_embedder, HttpLlmProvider};
 use std::sync::Arc;
 
-/// Probe the embedder for its output dimension unless explicitly configured.
+/// Resolve the embedding dimension for storage *without* probing the embedder.
 ///
-/// This is silent on purpose: the MCP frontend writes JSON-RPC to stdout, so
-/// no logging may happen here.
-async fn detect_dimension(config: &Config) -> Result<usize, Box<dyn std::error::Error>> {
+/// Order: an explicit dimension in the config (env `MEMAYU_EMBEDDER_DIM`, or the
+/// `embedding_dim` written to the config file by `memayu setup`) wins; otherwise
+/// the dimension already stored in the database schema is read back — a local
+/// read, never an embedder call. Fails only when neither is available (a fresh,
+/// unconfigured store), directing the user to `memayu setup`.
+///
+/// This is silent on purpose: the MCP frontend writes JSON-RPC to stdout, so no
+/// logging may happen here.
+pub async fn resolve_storage_dimension(
+    config: &Config,
+) -> Result<usize, Box<dyn std::error::Error>> {
     if let Some(dim) = config.dimension {
         return Ok(dim);
     }
-    let embedder = build_embedder(&config.embedder);
-    Ok(embedder.embed("dimension probe").await?.len())
+    let stored = match config.storage.backend {
+        StorageBackend::Libsql => {
+            memayu_storage_libsql::LibsqlProvider::inspect(&config.storage.libsql_path)
+                .await?
+                .dimension
+        }
+        StorageBackend::Postgres => {
+            memayu_storage_postgres::PostgresProvider::inspect(
+                config
+                    .storage
+                    .database_url
+                    .as_deref()
+                    .ok_or("missing postgres url")?,
+            )
+            .await?
+            .dimension
+        }
+    };
+    stored.ok_or_else(|| {
+        "no embedding dimension available for storage; run `memayu setup` or set MEMAYU_EMBEDDER_DIM"
+            .into()
+    })
 }
 
 /// Open a [`StorageProvider`] for the configured backend.
@@ -46,13 +74,13 @@ async fn build_storage(
 
 /// Build a [`MemoryService`] backed by static providers from the config.
 ///
-/// Returns the detected embedding dimension alongside the service. Used by the
+/// Returns the resolved embedding dimension alongside the service. Used by the
 /// TUI and local MCP frontends, which have no runtime dashboard to reconfigure
 /// providers.
 pub async fn build_service(
     config: &Config,
 ) -> Result<(Arc<MemoryService>, usize), Box<dyn std::error::Error>> {
-    let dimension = detect_dimension(config).await?;
+    let dimension = resolve_storage_dimension(config).await?;
     let storage = build_storage(config, dimension).await?;
     let service = Arc::new(
         MemoryService::new(