Added sqlite-faiss - an OGX-compatible BYOK database format

.tekton/integration-tests/pipeline/rag-content-0-8-integration-test.yaml

@@ -8,7 +8,8 @@ spec:
     Integration test: validates that rag-content images (CPU and CUDA) can
     generate a FAISS vector DB that lightspeed-stack can serve end-to-end.
     Runs on multiple architectures (x86_64 and arm64) using the Konflux
-    multi-platform controller. Tests all combinations: 2 images × 2 arches.
+    multi-platform controller. Tests all combinations: 2 images × 2 arches
+    × 2 vector stores (llamastack-faiss and sqlite-faiss).
   params:
     - name: SNAPSHOT
       description: 'JSON string with the application snapshot (contains rag-content-cpu-0-8, rag-content-cuda-12-9-0-8, and lightspeed-stack-0-8 components).'
@@ -127,7 +128,7 @@ spec:
               echo "========== End parameters =========="
 
     - name: run-integration-test
-      description: Run rag-content → lightspeed-stack E2E validation (platforms × images)
+      description: Run rag-content → lightspeed-stack E2E validation (platforms × images × vector stores)
       runAfter:
         - echo-integration-params
       matrix:
@@ -139,6 +140,10 @@ spec:
             value:
               - $(tasks.init-snapshot.results.cpu-image)
               - $(tasks.init-snapshot.results.cuda-image)
+          - name: VECTOR_STORE
+            value:
+              - llamastack-faiss
+              - sqlite-faiss
       params:
         - name: cmd
           value: |
@@ -155,6 +160,8 @@ spec:
             description: The platform of the VM to provision
           - name: RAG_CONTENT_IMAGE
             description: The rag-content container image to test
+          - name: VECTOR_STORE
+            description: Vector store type passed to generate_embeddings.py -s
           - name: cmd
             description: The command to run on the VM
           - name: envs
@@ -192,6 +199,8 @@ spec:
             env:
               - name: RAG_CONTENT_IMAGE
                 value: $(params.RAG_CONTENT_IMAGE)
+              - name: VECTOR_STORE
+                value: $(params.VECTOR_STORE)
               - name: TEST_CMD
                 value: $(params.cmd)
               - name: RESULTS_TEST_OUTPUT_PATH
@@ -263,8 +272,8 @@ spec:
               (cd repo && git fetch origin "${REPO_REV:-main}" && git checkout -q "${REPO_REV:-main}")
               mkdir -p scripts
 
-              # Build podman env flags (RAG_CONTENT_IMAGE comes from task param, not envs)
-              PODMAN_ENV=("-e" "RAG_CONTENT_IMAGE=$RAG_CONTENT_IMAGE")
+              # Build podman env flags (RAG_CONTENT_IMAGE and VECTOR_STORE come from task params)
+              PODMAN_ENV=("-e" "RAG_CONTENT_IMAGE=$RAG_CONTENT_IMAGE" "-e" "VECTOR_STORE=$VECTOR_STORE")
               while [ $# -ne 0 ]; do
                 PODMAN_ENV+=("-e" "$1")
                 shift
@@ -325,9 +334,15 @@ spec:
               exit $EXIT_CODE
 
     - name: run-gpu-integration-test
-      description: Run CUDA rag-content image on a GPU VM to verify GPU is actually used
+      description: Run CUDA rag-content image on a GPU VM (llamastack-faiss and sqlite-faiss)
       runAfter:
         - echo-integration-params
+      matrix:
+        params:
+          - name: VECTOR_STORE
+            value:
+              - llamastack-faiss
+              - sqlite-faiss
       params:
         - name: PLATFORM
           value: linux-g64xlarge/amd64
@@ -349,6 +364,8 @@ spec:
             description: The GPU platform of the VM to provision
           - name: RAG_CONTENT_IMAGE
             description: The CUDA rag-content container image to test
+          - name: VECTOR_STORE
+            description: Vector store type passed to generate_embeddings.py -s
           - name: cmd
             description: The command to run on the VM
           - name: envs
@@ -386,6 +403,8 @@ spec:
             env:
               - name: RAG_CONTENT_IMAGE
                 value: $(params.RAG_CONTENT_IMAGE)
+              - name: VECTOR_STORE
+                value: $(params.VECTOR_STORE)
               - name: TEST_CMD
                 value: $(params.cmd)
               - name: RESULTS_TEST_OUTPUT_PATH
@@ -470,6 +489,7 @@ spec:
               SCRIPTEOF
               # Inject env vars
               printf 'export RAG_CONTENT_IMAGE=%q\n' "$RAG_CONTENT_IMAGE" >> scripts/test.sh
+              printf 'export VECTOR_STORE=%q\n' "$VECTOR_STORE" >> scripts/test.sh
               while [ $# -ne 0 ]; do
                 VAR_NAME="${1%%=*}"
                 VAR_VALUE="${1#*=}"

README.md

@@ -90,7 +90,8 @@ You can generate the vector database either using
 1. [Llama-Index Faiss Vector Store](#faiss-vector-store)
 2. [Llama-Index Postgres (PGVector) Vector Store](#postgres-pgvector-vector-store)
 3. [OGX Faiss Vector-IO](#ogx-faiss)
-4. [OGX Postgres (PGVector) Vector Store](#ogx-postgres-pgvector-vector-store)
+4. [sqlite-faiss](#sqlite-faiss)
+5. [OGX Postgres (PGVector) Vector Store](#ogx-postgres-pgvector-vector-store)
 
 Each section below shows commands for both **local (uv)** and **container image (podman)** workflows. The container image already includes the default embedding model and all dependencies — no local Python setup required.
 
@@ -488,6 +489,64 @@ uv run python scripts/query_rag.py \
   -q "how can I configure a cinder backend"
 ```
 
+### sqlite-faiss
+
+`sqlite-faiss` writes the same `faiss_store.db` layout as `llamastack-faiss`
+(namespaced SQLite KV, `IndexFlatL2`) using FAISS and SentenceTransformer. The
+output is a drop-in BYOK file for Lightspeed Core Stack (`backend: faiss`).
+
+**Via uv (local):**
+
+```bash
+uv run ./custom_processor.py \
+  -o ./vector_db/custom_docs/0.1 \
+  -f ./custom_docs/0.1/ \
+  -md embeddings_model/ \
+  -mn sentence-transformers/all-mpnet-base-v2 \
+  -i custom_docs-0_1 \
+  --vector-store-type=sqlite-faiss
+```
+
+Or with the bundled generator:
+
+```bash
+uv run python scripts/generate_embeddings.py \
+  -f ./custom_docs/0.1 \
+  -o ./vector_db/custom_docs/0.1 \
+  -i custom_docs-0_1 \
+  -s sqlite-faiss
+```
+
+**Via podman:**
+
+```bash
+podman run --rm \
+  -v ./custom_docs:/rag-content/custom_docs:Z \
+  -v ./vector_db:/rag-content/vector_db:Z \
+  -v ./custom_processor.py:/rag-content/custom_processor.py:Z \
+  quay.io/lightspeed-core/rag-content-cpu:latest \
+  python ./custom_processor.py \
+    -o ./vector_db/custom_docs/0.1 \
+    -f ./custom_docs/0.1/ \
+    -md ./embeddings_model/ \
+    -mn sentence-transformers/all-mpnet-base-v2 \
+    -i custom_docs-0_1 \
+    --vector-store-type=sqlite-faiss
+```
+
+The output directory contains `faiss_store.db` and `lightspeed-stack.yaml`. There is
+no `llama-stack.yaml`. Query it with:
+
+```bash
+uv run python scripts/query_rag.py \
+  -p vector_db/custom_docs/0.1 \
+  -x custom-docs-0_1 \
+  -m embeddings_model \
+  -k 5 \
+  -q "how can I configure a cinder backend" \
+  --vector-store-type sqlite-faiss
+```
+
 ### OGX Postgres (PGVector) Vector Store
 
 To generate a vector database stored in Postgres (PGVector) for OGX, run the following

scripts/konflux_resolve.py

@@ -1195,6 +1195,17 @@ def _write_hashed_file_via_uv(
             os.remove(tmp_input)
 
 
+def _configure_logging(*, verbose: bool, quiet: bool) -> None:
+    """Set the process-wide log level from CLI flags."""
+    if verbose:
+        level = logging.DEBUG
+    elif quiet:
+        level = logging.ERROR
+    else:
+        level = logging.INFO
+    logging.basicConfig(level=level)
+
+
 def main() -> None:
     """Resolve dependencies with RHOAI-first policy and write Hermeto output files."""
     parser = argparse.ArgumentParser(
@@ -1205,13 +1216,7 @@ def main() -> None:
     verbosity.add_argument("--verbose", action="store_true", help="Verbose logging")
     verbosity.add_argument("--quiet", action="store_true", help="Errors only")
     args = parser.parse_args()
-
-    if args.verbose:
-        logging.basicConfig(level=logging.DEBUG)
-    elif args.quiet:
-        logging.basicConfig(level=logging.ERROR)
-    else:
-        logging.basicConfig(level=logging.INFO)
+    _configure_logging(verbose=args.verbose, quiet=args.quiet)
 
     profiles_path = os.path.join(KONFLUX_DIR, "profiles.toml")
     config = load_config(profiles_path, args.profile)
@@ -1220,7 +1225,6 @@ def main() -> None:
     wheel_only = load_wheel_only(wheel_only_path)
 
     python_version = config["python_version"]
-    platforms = config["platforms"]
     rhoai_index_url = config["rhoai_index_url"]
     suffix = config.get("output_suffix", "")
     tekton_files = config.get("tekton_files", [])

scripts/query_rag.py

@@ -255,6 +255,105 @@ def _ogx_query(args: argparse.Namespace) -> None:  # noqa: C901
                 print(f"Chunk ID: {chunk_id}\nScore: {score}\nText:\n{_get_chunk_text(chunk)}")
 
 
+def _sqlite_faiss_vector_store_id(db_dir: str, db_file: str) -> str:
+    """Resolve the vector_store_id from lightspeed-stack.yaml or the SQLite keys."""
+    lcs_path = os.path.join(db_dir, "lightspeed-stack.yaml")
+    if os.path.exists(lcs_path):
+        with open(lcs_path, encoding="utf-8") as fd:
+            cfg = yaml.safe_load(fd) or {}
+        stores = cfg.get("rag", {}).get("byok", {}).get("stores", [])
+        if stores and stores[0].get("vector_db_id"):
+            return str(stores[0]["vector_db_id"])
+
+    from lightspeed_rag_content.sqlite_faiss import list_sqlite_faiss_vector_store_ids
+
+    ids = list_sqlite_faiss_vector_store_ids(db_file)
+    if not ids:
+        raise ValueError(f"No sqlite-faiss vector store found in {db_file}")
+    return ids[0]
+
+
+def _sqlite_faiss_query(args: argparse.Namespace) -> None:
+    """Query a faiss_store.db written by sqlite-faiss."""
+    from sentence_transformers import SentenceTransformer
+
+    from lightspeed_rag_content.sqlite_faiss import search_sqlite_faiss_store
+
+    db_file = os.path.join(args.db_path, "faiss_store.db")
+    if not os.path.exists(db_file):
+        logging.error("Cannot find faiss_store.db in %s", args.db_path)
+        exit(1)
+
+    try:
+        vector_store_id = _sqlite_faiss_vector_store_id(args.db_path, db_file)
+    except ValueError as exc:
+        logging.error("%s", exc)
+        exit(1)
+
+    model = SentenceTransformer(os.path.realpath(args.model_path))
+    query_embedding = model.encode([args.query])[0].tolist()
+    hits = search_sqlite_faiss_store(db_file, vector_store_id, query_embedding, k=args.top_k)
+
+    if not hits:
+        logging.warning("No chunks retrieved for query: %s", args.query)
+        if args.json:
+            print(
+                json.dumps(
+                    {
+                        "query": args.query,
+                        "top_k": args.top_k,
+                        "threshold": args.threshold,
+                        "nodes": [],
+                    },
+                    indent=2,
+                )
+            )
+        exit(1)
+
+    if args.threshold > 0.0 and hits[0]["score"] > args.threshold:
+        logging.warning(
+            "Score %s of the top retrieved node for query '%s' "
+            "didn't cross the minimal threshold %s.",
+            hits[0]["score"],
+            args.query,
+            args.threshold,
+        )
+        if args.json:
+            print(
+                json.dumps(
+                    {
+                        "query": args.query,
+                        "top_k": args.top_k,
+                        "threshold": args.threshold,
+                        "nodes": [],
+                    },
+                    indent=2,
+                )
+            )
+        exit(1)
+
+    result = {
+        "query": args.query,
+        "top_k": args.top_k,
+        "threshold": args.threshold,
+        "nodes": [
+            {
+                "id": hit["chunk_id"],
+                "score": hit["score"],
+                "text": hit["content"],
+                "metadata": hit.get("metadata", {}),
+            }
+            for hit in hits
+        ],
+    }
+    if args.json:
+        print(json.dumps(result, indent=2))
+        return
+    for hit in hits:
+        print("=" * 80)
+        print(f"Chunk ID: {hit['chunk_id']}\nScore: {hit['score']}\nText:\n{hit['content']}")
+
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Utility script for querying RAG database")
     parser.add_argument(
@@ -278,7 +377,7 @@ def _ogx_query(args: argparse.Namespace) -> None:  # noqa: C901
     parser.add_argument(
         "--vector-store-type",
         default="auto",
-        choices=["auto", "faiss", "llamastack-faiss"],
+        choices=["auto", "faiss", "llamastack-faiss", "sqlite-faiss"],
         help="vector store type to be used.",
     )
     parser.add_argument(
@@ -310,12 +409,17 @@ def _ogx_query(args: argparse.Namespace) -> None:  # noqa: C901
         elif os.path.exists(os.path.join(args.db_path, "metadata.json")):
             args.vector_store_type = "faiss"
         elif os.path.exists(os.path.join(args.db_path, "faiss_store.db")):
-            args.vector_store_type = "llamastack-faiss"
+            if os.path.exists(os.path.join(args.db_path, "llama-stack.yaml")):
+                args.vector_store_type = "llamastack-faiss"
+            else:
+                args.vector_store_type = "sqlite-faiss"
         else:
             logging.error(f"Cannot recognize the DB in {args.db_path}")
             exit(1)
 
     if args.vector_store_type == "faiss":
         _llama_index_query(args)
+    elif args.vector_store_type == "sqlite-faiss":
+        _sqlite_faiss_query(args)
     else:
         _ogx_query(args)

scripts/remove_pytorch_cpu_pyproject.py

@@ -36,7 +36,7 @@ def remove_sections(file_path: str, sections_to_remove: list[str]) -> None:
         for key in keys[:-1]:
             if key not in current:
                 break
-            current = current[key]  # type: ignore
+            current = current[key]
         else:
             current.pop(keys[-1], None)
 

src/lightspeed_rag_content/config_templates.py

@@ -188,3 +188,27 @@
       sources:
         - {index_id}
 """
+
+
+def write_lcs_config_file(
+    filename: str,
+    *,
+    llama_stack_config_path: str,
+    byok_template: str,
+    index_id: str,
+    model_name: str,
+    dimension: object,
+    vector_store_id: str,
+    db_path: str,
+) -> None:
+    """Write lightspeed-stack.yaml from the base template plus a BYOK snippet."""
+    base = LCS_BASE_TEMPLATE.format(llama_stack_config_path=llama_stack_config_path)
+    data = base + byok_template.format(
+        index_id=index_id,
+        model_name=model_name,
+        dimension=dimension,
+        vector_store_id=vector_store_id,
+        db_path=db_path,
+    )
+    with open(filename, "w", encoding="utf-8") as fd:
+        fd.write(data)