From 6a1a0dede33d0c64d4c3979ba088876432fbf326 Mon Sep 17 00:00:00 2001
From: zhengwj533 <99308043+zhengwj533@users.noreply.github.com>
Date: Tue, 3 Jun 2025 09:31:19 +0800
Subject: [PATCH 01/10] add document type

---
 py/shared/abstractions/document.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/py/shared/abstractions/document.py b/py/shared/abstractions/document.py
index eecfdde82..3c230a3e1 100644
--- a/py/shared/abstractions/document.py
+++ b/py/shared/abstractions/document.py
@@ -80,6 +80,7 @@ class DocumentType(str, Enum):
 
     # Video/GIF
     GIF = "gif"
+    MP4 = "mp4"
 
     # Word
     DOC = "doc"
@@ -91,6 +92,9 @@ class DocumentType(str, Enum):
     TS = "ts"
     CSS = "css"
 
+    # other type
+    UNKNOWN = "UNKNOWN"
+
 
 class Document(R2RSerializable):
     id: UUID = Field(default_factory=uuid4)

From 99d1454bc2109736987a7d1d1585c4c212bc766d Mon Sep 17 00:00:00 2001
From: zhengwj <zhengwj@neusoft.com>
Date: Wed, 4 Jun 2025 09:44:29 +0800
Subject: [PATCH 02/10] add underlying parse

---
 py/core/__init__.py                     |   1 +
 py/core/parsers/__init__.py             |   1 +
 py/core/parsers/media/__init__.py       |   2 +
 py/core/parsers/media/video_parser.py   | 139 ++++++++++++++++++++++++
 py/core/providers/ingestion/r2r/base.py |  11 ++
 py/shared/abstractions/document.py      |  10 +-
 6 files changed, 160 insertions(+), 4 deletions(-)
 create mode 100644 py/core/parsers/media/video_parser.py

diff --git a/py/core/__init__.py b/py/core/__init__.py
index c27da75af..dda7581db 100644
--- a/py/core/__init__.py
+++ b/py/core/__init__.py
@@ -121,6 +121,7 @@
     "RetrievalService",
     "GraphService",
     "AudioParser",
+    "VideoParser",
     "BMPParser",
     "DOCParser",
     "DOCXParser",
diff --git a/py/core/parsers/__init__.py b/py/core/parsers/__init__.py
index cff1c74dc..8d259cb42 100644
--- a/py/core/parsers/__init__.py
+++ b/py/core/parsers/__init__.py
@@ -4,6 +4,7 @@
 
 __all__ = [
     "AudioParser",
+    "VideoParser",
     "BMPParser",
     "DOCParser",
     "DOCXParser",
diff --git a/py/core/parsers/media/__init__.py b/py/core/parsers/media/__init__.py
index 2ece41197..10cdfb4ee 100644
--- a/py/core/parsers/media/__init__.py
+++ b/py/core/parsers/media/__init__.py
@@ -1,5 +1,6 @@
 # type: ignore
 from .audio_parser import AudioParser
+from .video_parser import VideoParser
 from .bmp_parser import BMPParser
 from .doc_parser import DOCParser
 from .docx_parser import DOCXParser
@@ -17,6 +18,7 @@
 
 __all__ = [
     "AudioParser",
+    "VideoParser",
     "BMPParser",
     "DOCParser",
     "DOCXParser",
diff --git a/py/core/parsers/media/video_parser.py b/py/core/parsers/media/video_parser.py
new file mode 100644
index 000000000..57e7984e4
--- /dev/null
+++ b/py/core/parsers/media/video_parser.py
@@ -0,0 +1,139 @@
+import logging
+from typing import AsyncGenerator
+
+from core.base.abstractions import GenerationConfig
+from core.base.parsers.base_parser import AsyncParser
+from core.base.providers import (
+    CompletionProvider,
+    DatabaseProvider,
+    IngestionConfig,
+)
+from core.providers import PostgresDatabaseProvider
+
+logger = logging.getLogger(__name__)
+
+
+class VideoParser(AsyncParser[str | bytes | dict]):
+    """
+    Video parser that processes video files for ingestion.
+
+    This parser handles:
+    - Video content analysis using LLM
+    """
+
+    # Mapping of file extensions to MIME types
+    MIME_TYPE_MAPPING = {
+        "mp4": "video/mp4",
+        "avi": "video/avi",
+        "mov": "video/quicktime",
+        "mkv": "video/x-matroska",
+    }
+
+    def __init__(
+        self,
+        config: IngestionConfig,
+        database_provider: DatabaseProvider,
+        llm_provider: CompletionProvider,
+    ):
+        """
+        Initialize the video parser.
+
+        Sets up providers and configurations needed for video processing.
+        """
+        super().__init__()
+
+        self.config = config
+        self.database_provider: PostgresDatabaseProvider = database_provider
+        self.llm_provider = llm_provider
+
+        self.video_prompt_name = self.config.extra_fields.get(
+            "extra_video_prompt_name", "vision_img"
+        )
+        self.video_prompt_args = self.config.extra_fields.get(
+            "extra_video_prompt_args", {}
+        )
+
+        logger.info(
+            "Video parser initialized with default prompt template: %s",
+            self.video_prompt_name,
+        )
+
+    async def ingest(
+        self, data: str | bytes | dict, **kwargs
+    ) -> AsyncGenerator[str, None]:
+        """
+        Process video file for ingestion.
+
+        Args:
+            data: The video data to process, str or dict containing the URL of the video file
+                file_url: Optional URL of the video file
+            **kwargs:
+                file_url: Optional URL of the video file
+                vlm: Optional model to use for processing
+                prompt_name: Optional name of the prompt to use
+                prompt_args: Optional arguments for the prompt
+
+        Yields:
+            str: Generated descriptions from video and audio analysis
+        """
+        if isinstance(data, dict):
+            file_url = data.get("file_url")
+            if not file_url:
+                file_url = kwargs.get("file_url")
+
+        logger.debug("file for ingest: %s", file_url)
+        if file_url is None:
+            raise ValueError("file_url is required")
+
+        # Process video chunks
+        async for description in self._call_llm(file_url, **kwargs):
+            yield description
+
+    async def _call_llm(self, file_url, **kwargs) -> AsyncGenerator[str, None]:
+
+        model = kwargs.get("vlm", self.config.app.vlm)
+        generation_config = GenerationConfig(
+            model=model,
+            stream=False,
+        )
+
+        # Load prompt texts
+        prompt_name = kwargs.get("prompt_name", self.video_prompt_name)
+        prompt_args = kwargs.get("prompt_args", self.video_prompt_args)
+        video_prompt_text = (
+            await self.database_provider.prompts_handler.get_cached_prompt(
+                prompt_name=prompt_name,
+                inputs=prompt_args,
+            )
+        )
+
+        messages = [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": video_prompt_text},
+                    {
+                        "type": "video_url",
+                        "video_url": {"url": file_url},
+                    },
+                ],
+            }
+        ]
+
+        try:
+            response = await self.llm_provider.aget_completion(
+                messages=messages, generation_config=generation_config
+            )
+
+            if not response.choices or not response.choices[0].message:
+                raise ValueError("No response content")
+
+            content = response.choices[0].message.content
+            if not content:
+                raise ValueError("Empty response content")
+
+            yield content
+
+        except Exception as e:
+            logger.error(f"Error processing file {file_url}: {str(e)}")
+            raise
diff --git a/py/core/providers/ingestion/r2r/base.py b/py/core/providers/ingestion/r2r/base.py
index 77dda2254..79e9ad0a3 100644
--- a/py/core/providers/ingestion/r2r/base.py
+++ b/py/core/providers/ingestion/r2r/base.py
@@ -64,6 +64,10 @@ class R2RIngestionProvider(IngestionProvider):
         DocumentType.HEIC: parsers.ImageParser,
         DocumentType.SVG: parsers.ImageParser,
         DocumentType.MP3: parsers.AudioParser,
+        DocumentType.MP4: parsers.VideoParser,
+        DocumentType.AVI: parsers.VideoParser,
+        DocumentType.MOV: parsers.VideoParser,
+        DocumentType.MKV: parsers.VideoParser,
         DocumentType.P7S: parsers.P7SParser,
         DocumentType.RST: parsers.RSTParser,
         DocumentType.RTF: parsers.RTFParser,
@@ -93,6 +97,13 @@ class R2RIngestionProvider(IngestionProvider):
         DocumentType.PNG,
         DocumentType.SVG,
     }
+    
+    VIDEO_TYPES = {
+        DocumentType.MP4,
+        DocumentType.AVI,
+        DocumentType.MOV,
+        DocumentType.MKV,
+    }
 
     def __init__(
         self,
diff --git a/py/shared/abstractions/document.py b/py/shared/abstractions/document.py
index 3c230a3e1..87bbf61aa 100644
--- a/py/shared/abstractions/document.py
+++ b/py/shared/abstractions/document.py
@@ -20,6 +20,12 @@ class DocumentType(str, Enum):
 
     # Audio
     MP3 = "mp3"
+    
+    # Video
+    MP4 = "mp4"
+    AVI = "avi"
+    MOV = "mov"
+    MKV = "mkv"
 
     # CSV
     CSV = "csv"
@@ -80,7 +86,6 @@ class DocumentType(str, Enum):
 
     # Video/GIF
     GIF = "gif"
-    MP4 = "mp4"
 
     # Word
     DOC = "doc"
@@ -92,9 +97,6 @@ class DocumentType(str, Enum):
     TS = "ts"
     CSS = "css"
 
-    # other type
-    UNKNOWN = "UNKNOWN"
-
 
 class Document(R2RSerializable):
     id: UUID = Field(default_factory=uuid4)

From 9afb692e7607a2615b702fc7e7a744da570ebe46 Mon Sep 17 00:00:00 2001
From: zhengwj533 <99308043+zhengwj533@users.noreply.github.com>
Date: Wed, 4 Jun 2025 14:34:21 +0800
Subject: [PATCH 03/10] use ingestionconfig.vlm

---
 py/core/parsers/media/video_parser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/py/core/parsers/media/video_parser.py b/py/core/parsers/media/video_parser.py
index 57e7984e4..fbce3e5a6 100644
--- a/py/core/parsers/media/video_parser.py
+++ b/py/core/parsers/media/video_parser.py
@@ -91,7 +91,7 @@ async def ingest(
 
     async def _call_llm(self, file_url, **kwargs) -> AsyncGenerator[str, None]:
 
-        model = kwargs.get("vlm", self.config.app.vlm)
+        model = kwargs.get("vlm", self.config.vlm)
         generation_config = GenerationConfig(
             model=model,
             stream=False,

From 40894a592589e864fe725b96650caa3d893336a0 Mon Sep 17 00:00:00 2001
From: zhengwj533 <99308043+zhengwj533@users.noreply.github.com>
Date: Wed, 4 Jun 2025 14:41:44 +0800
Subject: [PATCH 04/10] fix mypy error

---
 py/core/parsers/media/video_parser.py | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/py/core/parsers/media/video_parser.py b/py/core/parsers/media/video_parser.py
index fbce3e5a6..7a57b0deb 100644
--- a/py/core/parsers/media/video_parser.py
+++ b/py/core/parsers/media/video_parser.py
@@ -43,7 +43,7 @@ def __init__(
         super().__init__()
 
         self.config = config
-        self.database_provider: PostgresDatabaseProvider = database_provider
+        self.database_provider = database_provider
         self.llm_provider = llm_provider
 
         self.video_prompt_name = self.config.extra_fields.get(
@@ -59,7 +59,7 @@ def __init__(
         )
 
     async def ingest(
-        self, data: str | bytes | dict, **kwargs
+        self, data: str | bytes, **kwargs
     ) -> AsyncGenerator[str, None]:
         """
         Process video file for ingestion.
@@ -76,11 +76,7 @@ async def ingest(
         Yields:
             str: Generated descriptions from video and audio analysis
         """
-        if isinstance(data, dict):
-            file_url = data.get("file_url")
-            if not file_url:
-                file_url = kwargs.get("file_url")
-
+        file_url = kwargs.get("file_url")
         logger.debug("file for ingest: %s", file_url)
         if file_url is None:
             raise ValueError("file_url is required")

From 31b8c2019eea9bb357552607521b14bb343cf111 Mon Sep 17 00:00:00 2001
From: zhengwj533 <99308043+zhengwj533@users.noreply.github.com>
Date: Wed, 4 Jun 2025 14:52:59 +0800
Subject: [PATCH 05/10] fix

---
 py/core/parsers/media/video_parser.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/py/core/parsers/media/video_parser.py b/py/core/parsers/media/video_parser.py
index 7a57b0deb..937b8259e 100644
--- a/py/core/parsers/media/video_parser.py
+++ b/py/core/parsers/media/video_parser.py
@@ -59,7 +59,7 @@ def __init__(
         )
 
     async def ingest(
-        self, data: str | bytes, **kwargs
+        self, data: str | bytes | dict, **kwargs
     ) -> AsyncGenerator[str, None]:
         """
         Process video file for ingestion.
@@ -76,7 +76,10 @@ async def ingest(
         Yields:
             str: Generated descriptions from video and audio analysis
         """
-        file_url = kwargs.get("file_url")
+        if isinstance(data, dict):
+            file_url = data.get("file_url")
+            if not file_url:
+                file_url = kwargs.get("file_url")
         logger.debug("file for ingest: %s", file_url)
         if file_url is None:
             raise ValueError("file_url is required")

From f10a1ba09147775f81aaf7b1db493e4f909c58de Mon Sep 17 00:00:00 2001
From: zhengwj533 <99308043+zhengwj533@users.noreply.github.com>
Date: Wed, 4 Jun 2025 15:06:13 +0800
Subject: [PATCH 06/10] fix

---
 py/core/parsers/media/video_parser.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/py/core/parsers/media/video_parser.py b/py/core/parsers/media/video_parser.py
index 937b8259e..f617743de 100644
--- a/py/core/parsers/media/video_parser.py
+++ b/py/core/parsers/media/video_parser.py
@@ -84,11 +84,11 @@ async def ingest(
         if file_url is None:
             raise ValueError("file_url is required")
 
-        # Process video chunks
-        async for description in self._call_llm(file_url, **kwargs):
-            yield description
+        # Process video
+        description = self._call_llm(file_url, **kwargs)
+        yield description
 
-    async def _call_llm(self, file_url, **kwargs) -> AsyncGenerator[str, None]:
+    async def _call_llm(self, file_url, **kwargs) -> str:
 
         model = kwargs.get("vlm", self.config.vlm)
         generation_config = GenerationConfig(
@@ -131,7 +131,7 @@ async def _call_llm(self, file_url, **kwargs) -> AsyncGenerator[str, None]:
             if not content:
                 raise ValueError("Empty response content")
 
-            yield content
+            return content
 
         except Exception as e:
             logger.error(f"Error processing file {file_url}: {str(e)}")

From 84585bf09d024899703412ca27c37bde94eeeb5f Mon Sep 17 00:00:00 2001
From: zhengwj533 <99308043+zhengwj533@users.noreply.github.com>
Date: Wed, 4 Jun 2025 15:28:15 +0800
Subject: [PATCH 07/10] fix mypy error with ignore

---
 py/core/parsers/media/video_parser.py | 23 +++++++++--------------
 1 file changed, 9 insertions(+), 14 deletions(-)

diff --git a/py/core/parsers/media/video_parser.py b/py/core/parsers/media/video_parser.py
index f617743de..8cbf5ea5f 100644
--- a/py/core/parsers/media/video_parser.py
+++ b/py/core/parsers/media/video_parser.py
@@ -8,7 +8,6 @@
     DatabaseProvider,
     IngestionConfig,
 )
-from core.providers import PostgresDatabaseProvider
 
 logger = logging.getLogger(__name__)
 
@@ -47,7 +46,7 @@ def __init__(
         self.llm_provider = llm_provider
 
         self.video_prompt_name = self.config.extra_fields.get(
-            "extra_video_prompt_name", "vision_img"
+            "extra_video_prompt_name", "vision_video"
         )
         self.video_prompt_args = self.config.extra_fields.get(
             "extra_video_prompt_args", {}
@@ -58,7 +57,7 @@ def __init__(
             self.video_prompt_name,
         )
 
-    async def ingest(
+    async def ingest(  # type: ignore[override]
         self, data: str | bytes | dict, **kwargs
     ) -> AsyncGenerator[str, None]:
         """
@@ -80,16 +79,14 @@ async def ingest(
             file_url = data.get("file_url")
             if not file_url:
                 file_url = kwargs.get("file_url")
+        else:
+            file_url = kwargs.get("file_url")
+
         logger.debug("file for ingest: %s", file_url)
         if file_url is None:
             raise ValueError("file_url is required")
 
         # Process video
-        description = self._call_llm(file_url, **kwargs)
-        yield description
-
-    async def _call_llm(self, file_url, **kwargs) -> str:
-
         model = kwargs.get("vlm", self.config.vlm)
         generation_config = GenerationConfig(
             model=model,
@@ -99,11 +96,9 @@ async def _call_llm(self, file_url, **kwargs) -> str:
         # Load prompt texts
         prompt_name = kwargs.get("prompt_name", self.video_prompt_name)
         prompt_args = kwargs.get("prompt_args", self.video_prompt_args)
-        video_prompt_text = (
-            await self.database_provider.prompts_handler.get_cached_prompt(
-                prompt_name=prompt_name,
-                inputs=prompt_args,
-            )
+        video_prompt_text = await self.database_provider.prompts_handler.get_cached_prompt(  # type: ignore # noqa E501
+            prompt_name=prompt_name,
+            inputs=prompt_args,
         )
 
         messages = [
@@ -131,7 +126,7 @@ async def _call_llm(self, file_url, **kwargs) -> str:
             if not content:
                 raise ValueError("Empty response content")
 
-            return content
+            yield content
 
         except Exception as e:
             logger.error(f"Error processing file {file_url}: {str(e)}")

From ed66802df7d75c9b9d116d699ddf695b08e3c528 Mon Sep 17 00:00:00 2001
From: zhengwj <zhengwj@neusoft.com>
Date: Sun, 27 Jul 2025 21:19:37 +0800
Subject: [PATCH 08/10] add unit test

---
 README.md                                  |   2 +-
 py/core/parsers/media/__init__.py          |   2 +-
 py/core/parsers/media/video_parser.py      | 113 +++++++++++++--------
 py/core/providers/ingestion/r2r/base.py    |   2 +-
 py/shared/abstractions/document.py         |   2 +-
 py/tests/unit/parsers/test_video_parser.py | 102 +++++++++++++++++++
 6 files changed, 174 insertions(+), 49 deletions(-)
 create mode 100644 py/tests/unit/parsers/test_video_parser.py

diff --git a/README.md b/README.md
index e6ea2a26c..cbe34ac16 120000
--- a/README.md
+++ b/README.md
@@ -1 +1 @@
-./py/README.md
\ No newline at end of file
+./py/README.md
diff --git a/py/core/parsers/media/__init__.py b/py/core/parsers/media/__init__.py
index 10cdfb4ee..893115e03 100644
--- a/py/core/parsers/media/__init__.py
+++ b/py/core/parsers/media/__init__.py
@@ -1,6 +1,5 @@
 # type: ignore
 from .audio_parser import AudioParser
-from .video_parser import VideoParser
 from .bmp_parser import BMPParser
 from .doc_parser import DOCParser
 from .docx_parser import DOCXParser
@@ -15,6 +14,7 @@
 from .ppt_parser import PPTParser
 from .pptx_parser import PPTXParser
 from .rtf_parser import RTFParser
+from .video_parser import VideoParser
 
 __all__ = [
     "AudioParser",
diff --git a/py/core/parsers/media/video_parser.py b/py/core/parsers/media/video_parser.py
index 8cbf5ea5f..b7941ebcb 100644
--- a/py/core/parsers/media/video_parser.py
+++ b/py/core/parsers/media/video_parser.py
@@ -1,3 +1,4 @@
+import base64
 import logging
 from typing import AsyncGenerator
 
@@ -12,12 +13,9 @@
 logger = logging.getLogger(__name__)
 
 
-class VideoParser(AsyncParser[str | bytes | dict]):
+class VideoParser(AsyncParser[str | bytes]):
     """
-    Video parser that processes video files for ingestion.
-
-    This parser handles:
-    - Video content analysis using LLM
+    A parser for video files.
     """
 
     # Mapping of file extensions to MIME types
@@ -34,23 +32,14 @@ def __init__(
         database_provider: DatabaseProvider,
         llm_provider: CompletionProvider,
     ):
-        """
-        Initialize the video parser.
-
-        Sets up providers and configurations needed for video processing.
-        """
         super().__init__()
 
         self.config = config
         self.database_provider = database_provider
         self.llm_provider = llm_provider
 
-        self.video_prompt_name = self.config.extra_fields.get(
-            "extra_video_prompt_name", "vision_video"
-        )
-        self.video_prompt_args = self.config.extra_fields.get(
-            "extra_video_prompt_args", {}
-        )
+        self.video_prompt_name = "video_understanding"
+        self.video_prompt_args: dict = {}
 
         logger.info(
             "Video parser initialized with default prompt template: %s",
@@ -58,45 +47,77 @@ def __init__(
         )
 
     async def ingest(  # type: ignore[override]
-        self, data: str | bytes | dict, **kwargs
+        self, data: str | bytes, **kwargs
     ) -> AsyncGenerator[str, None]:
         """
         Process video file for ingestion.
 
         Args:
-            data: The video data to process, str or dict containing the URL of the video file
-                file_url: Optional URL of the video file
-            **kwargs:
-                file_url: Optional URL of the video file
-                vlm: Optional model to use for processing
-                prompt_name: Optional name of the prompt to use
-                prompt_args: Optional arguments for the prompt
+            data: The video data to process, file url or raw bytes.
+            **kwargs: Additional arguments:
+                - file_type: The type of the video file (e.g., "mp4", "avi").
+                - bytes_limit: Optional limit for raw bytes size.
+                - vlm: Optional vision model to use for processing.
+                - prompt_name: Optional name of the prompt template to use.
+                - input_args: Optional arguments for the prompt template.
 
         Yields:
             str: Generated descriptions from video and audio analysis
         """
-        if isinstance(data, dict):
-            file_url = data.get("file_url")
-            if not file_url:
-                file_url = kwargs.get("file_url")
-        else:
-            file_url = kwargs.get("file_url")
-
-        logger.debug("file for ingest: %s", file_url)
-        if file_url is None:
-            raise ValueError("file_url is required")
-
-        # Process video
-        model = kwargs.get("vlm", self.config.vlm)
+        file_type = kwargs.get("file_type")
+        if not file_type:
+            raise ValueError("file_type must be provided")
+        if file_type not in self.MIME_TYPE_MAPPING:
+            raise ValueError(
+                f"file type must be one of {list(self.MIME_TYPE_MAPPING.keys())}"
+            )
+        bytes_limit = kwargs.get(
+            "bytes_limit", 5 * 1024 * 1024
+        )  # Default to 5MB
+        if not isinstance(bytes_limit, int):
+            raise ValueError("bytes_limit must be an integer")
+
+        vlm = kwargs.get("vlm")
+        prompt_name = kwargs.get("prompt_name")
+        input_args = kwargs.get("input_args")
+        if isinstance(data, bytes):
+            if (
+                bytes_limit is None
+                or bytes_limit < 0
+                or bytes_limit > 5 * 1024 * 1024
+            ):
+                raise ValueError(
+                    "bytes_limit must be a positive integer up to 5MB"
+                )
+            if len(data) > bytes_limit:
+                raise ValueError(
+                    f"file raw bytes size must be less than {bytes_limit} bytes"
+                )
+
+        if isinstance(data, str):
+            url_or_base64 = data
+        elif isinstance(data, bytes):
+            base564str = base64.b64encode(data).decode("utf-8")
+            url_or_base64 = f"data:video/{file_type};base64,{base564str}"
+
+        model = vlm or self.config.vlm
         generation_config = GenerationConfig(
             model=model,
             stream=False,
         )
 
-        # Load prompt texts
-        prompt_name = kwargs.get("prompt_name", self.video_prompt_name)
-        prompt_args = kwargs.get("prompt_args", self.video_prompt_args)
-        video_prompt_text = await self.database_provider.prompts_handler.get_cached_prompt(  # type: ignore # noqa E501
+        prompt_name = prompt_name or self.video_prompt_name
+        prompt_args = input_args or self.video_prompt_args
+        prompts_handler = (
+            self.database_provider.prompts_handler
+            if hasattr(self.database_provider, "prompts_handler")
+            else None
+        )
+        if not prompts_handler:
+            raise ValueError(
+                "Prompts handler is not available in the provider"
+            )
+        video_prompt_text = await prompts_handler.get_cached_prompt(
             prompt_name=prompt_name,
             inputs=prompt_args,
         )
@@ -105,11 +126,11 @@ async def ingest(  # type: ignore[override]
             {
                 "role": "user",
                 "content": [
-                    {"type": "text", "text": video_prompt_text},
                     {
                         "type": "video_url",
-                        "video_url": {"url": file_url},
+                        "video_url": {"url": url_or_base64},
                     },
+                    {"type": "text", "text": video_prompt_text},
                 ],
             }
         ]
@@ -124,10 +145,12 @@ async def ingest(  # type: ignore[override]
 
             content = response.choices[0].message.content
             if not content:
-                raise ValueError("Empty response content")
+                raise ValueError("Response content is empty")
 
             yield content
 
         except Exception as e:
-            logger.error(f"Error processing file {file_url}: {str(e)}")
+            logger.error(
+                f"Error processing file {url_or_base64[:50]}: {str(e)}"
+            )
             raise
diff --git a/py/core/providers/ingestion/r2r/base.py b/py/core/providers/ingestion/r2r/base.py
index 79e9ad0a3..f7ad72c7d 100644
--- a/py/core/providers/ingestion/r2r/base.py
+++ b/py/core/providers/ingestion/r2r/base.py
@@ -97,7 +97,7 @@ class R2RIngestionProvider(IngestionProvider):
         DocumentType.PNG,
         DocumentType.SVG,
     }
-    
+
     VIDEO_TYPES = {
         DocumentType.MP4,
         DocumentType.AVI,
diff --git a/py/shared/abstractions/document.py b/py/shared/abstractions/document.py
index 87bbf61aa..fc861c6b4 100644
--- a/py/shared/abstractions/document.py
+++ b/py/shared/abstractions/document.py
@@ -20,7 +20,7 @@ class DocumentType(str, Enum):
 
     # Audio
     MP3 = "mp3"
-    
+
     # Video
     MP4 = "mp4"
     AVI = "avi"
diff --git a/py/tests/unit/parsers/test_video_parser.py b/py/tests/unit/parsers/test_video_parser.py
new file mode 100644
index 000000000..238039834
--- /dev/null
+++ b/py/tests/unit/parsers/test_video_parser.py
@@ -0,0 +1,102 @@
+import pytest
+import pytest_asyncio
+from unittest.mock import AsyncMock, MagicMock
+
+from core.parsers.media.video_parser import VideoParser
+
+
+class DummyApp:
+    vlm = "test-vlm"
+
+
+class DummyConfig:
+    app = DummyApp()
+
+
+@pytest_asyncio.fixture
+def mock_db_provider():
+    mock = MagicMock()
+    mock.prompts_handler.get_cached_prompt = AsyncMock(
+        return_value="prompt text"
+    )
+    return mock
+
+
+@pytest_asyncio.fixture
+def mock_llm_provider():
+    mock = MagicMock()
+    mock.aget_completion = AsyncMock(
+        return_value=MagicMock(
+            choices=[MagicMock(message=MagicMock(content="video description"))]
+        )
+    )
+    return mock
+
+
+@pytest.mark.asyncio
+async def test_ingest_str_success(mock_db_provider, mock_llm_provider):
+    parser = VideoParser(
+        config=DummyConfig(),
+        database_provider=mock_db_provider,
+        llm_provider=mock_llm_provider,
+    )
+    gen = parser.ingest(
+        "http://test/video.mp4", file_type="mp4", bytes_limit=None
+    )
+    result = [x async for x in gen]
+    assert result == ["video description"]
+
+
+@pytest.mark.asyncio
+async def test_ingest_bytes_success(mock_db_provider, mock_llm_provider):
+    parser = VideoParser(
+        config=DummyConfig(),
+        database_provider=mock_db_provider,
+        llm_provider=mock_llm_provider,
+    )
+    data = b"1234"
+    gen = parser.ingest(data, file_type="mp4", bytes_limit=10)
+    result = [x async for x in gen]
+    assert result == ["video description"]
+
+
+@pytest.mark.asyncio
+async def test_ingest_invalid_file_type(mock_db_provider, mock_llm_provider):
+    parser = VideoParser(
+        config=DummyConfig(),
+        database_provider=mock_db_provider,
+        llm_provider=mock_llm_provider,
+    )
+    with pytest.raises(ValueError):
+        gen = parser.ingest("http://test/video.xyz", file_type="xyz")
+        [x async for x in gen]
+
+
+@pytest.mark.asyncio
+async def test_ingest_bytes_limit_exceeded(
+    mock_db_provider, mock_llm_provider
+):
+    parser = VideoParser(
+        config=DummyConfig(),
+        database_provider=mock_db_provider,
+        llm_provider=mock_llm_provider,
+    )
+    data = b"1" * 11
+    with pytest.raises(ValueError):
+        gen = parser.ingest(data, file_type="mp4", bytes_limit=10)
+        [x async for x in gen]
+
+
+@pytest.mark.asyncio
+async def test_ingest_llm_no_response(mock_db_provider, mock_llm_provider):
+    mock_llm_provider.aget_completion = AsyncMock(
+        return_value=MagicMock(choices=[])
+    )
+    parser = VideoParser(
+        config=DummyConfig(),
+        database_provider=mock_db_provider,
+        llm_provider=mock_llm_provider,
+    )
+    gen = parser.ingest("http://test/video.mp4", file_type="mp4")
+    with pytest.raises(ValueError):
+        [x async for x in gen]

From 67f8927e2ece230257f12a5b77a6d3da81774106 Mon Sep 17 00:00:00 2001
From: zhengwj <zhengwj@neusoft.com>
Date: Sun, 27 Jul 2025 22:12:10 +0800
Subject: [PATCH 09/10] a sample prompt for video understanding

---
 .../database/prompts/vision_video.yaml        | 46 +++++++++++++++++++
 1 file changed, 46 insertions(+)
 create mode 100644 py/core/providers/database/prompts/vision_video.yaml

diff --git a/py/core/providers/database/prompts/vision_video.yaml b/py/core/providers/database/prompts/vision_video.yaml
new file mode 100644
index 000000000..3265b1564
--- /dev/null
+++ b/py/core/providers/database/prompts/vision_video.yaml
@@ -0,0 +1,46 @@
+video_understanding:
+  template: >
+    # Video-Understanding Prompt
+    
+    ## Objective  
+    Produce a text-only reconstruction of the video that allows a reader to experience it without ever pressing play.
+    
+    ---
+    
+    ## Output Structure (Markdown)
+    
+    ### 1. Title  
+    A single-line, evocative summary  
+    Example:  
+    `Neon-Drenched Midnight Ramen: A Slow-Motion Culinary Ode`
+    
+    ---
+    
+    ### 2. Scene-by-Scene Timeline  
+    | Timestamp | Visuals & Camera | Audio & Speech | On-Screen Text / Graphics | Mood |
+    |-----------|------------------|----------------|---------------------------|------|
+    | 00:00-00:02 | Fade-in from black; overhead establishing shot of a cramped Tokyo kitchen; warm tungsten glow | Low sizzle of pork fat, subtle city hum outside | White kanji on black: Episode 7 | Anticipatory, intimate |
+    | 00:02-00:07 | Hand-held camera glides forward; steam curls like incense above a rolling boil | Water bubbles crescendo; no dialogue | None | Focused, almost meditative |
+    | … | … | … | … | … |
+    
+    ---
+    
+    ### 3. Transcript (Verbatim)  
+    - 00:10-00:15  
+      Male narrator, deep & calm, Japanese with English subtitles:  
+      “In an alley off Shibuya, at 2 a.m., the last customer orders a bowl of tonkotsu.”
+    
+    ---
+    
+    ### 4. Environment & Object Inventory  
+    - Countertop: Stainless steel reflects overhead bulb; soy-sauce bottle labeled Yamasho in red calligraphy; digital timer shows 03:47.  
+    - Ingredients: Chashu cross-section reveals spiral fat marbling; green onions diced to 2 mm cubes.
+    
+    ---
+    
+    ### 5. Style & Technical Notes  
+    - Color palette: High-contrast warm tones—dominant reds & ambers, saturation ≈ 75 %.  
+    - Motion: 60 fps slowed to 30 fps; subtle handheld micro-shakes.  
+    - Soundtrack: Lo-fi hip-hop loop, 72 BPM, sparse piano chords every 8 bars.
+    
+  input_types: {}

From e9389d455744affba0a1d5703d351711acb5a23a Mon Sep 17 00:00:00 2001
From: zhengwj <zhengwj@neusoft.com>
Date: Mon, 28 Jul 2025 10:36:42 +0800
Subject: [PATCH 10/10] get settings from ingestion config

---
 py/core/parsers/media/video_parser.py         | 21 +++++---
 .../database/prompts/vision_video.yaml        | 52 +++++++++----------
 py/tests/unit/parsers/test_video_parser.py    | 11 ++--
 3 files changed, 46 insertions(+), 38 deletions(-)

diff --git a/py/core/parsers/media/video_parser.py b/py/core/parsers/media/video_parser.py
index b7941ebcb..048691e10 100644
--- a/py/core/parsers/media/video_parser.py
+++ b/py/core/parsers/media/video_parser.py
@@ -38,12 +38,16 @@ def __init__(
         self.database_provider = database_provider
         self.llm_provider = llm_provider
 
+        self.vlm = (
+            self.config.vlm or self.config.app.vlm if self.config.app else None
+        )
         self.video_prompt_name = "video_understanding"
         self.video_prompt_args: dict = {}
 
         logger.info(
-            "Video parser initialized with default prompt template: %s",
+            "Video parser initialized with default prompt template: %s and vlm: %s",
             self.video_prompt_name,
+            self.vlm,
         )
 
     async def ingest(  # type: ignore[override]
@@ -64,22 +68,23 @@ async def ingest(  # type: ignore[override]
         Yields:
             str: Generated descriptions from video and audio analysis
         """
-        file_type = kwargs.get("file_type")
+        extra_fields = kwargs.get("extra_fields", {})
+        file_type = extra_fields.get("file_type")
         if not file_type:
             raise ValueError("file_type must be provided")
         if file_type not in self.MIME_TYPE_MAPPING:
             raise ValueError(
                 f"file type must be one of {list(self.MIME_TYPE_MAPPING.keys())}"
             )
-        bytes_limit = kwargs.get(
+        bytes_limit = extra_fields.get(
             "bytes_limit", 5 * 1024 * 1024
         )  # Default to 5MB
         if not isinstance(bytes_limit, int):
             raise ValueError("bytes_limit must be an integer")
 
-        vlm = kwargs.get("vlm")
-        prompt_name = kwargs.get("prompt_name")
-        input_args = kwargs.get("input_args")
+        vlm = extra_fields.get("vlm")
+        prompt_name = extra_fields.get("prompt_name")
+        input_args = extra_fields.get("input_args")
         if isinstance(data, bytes):
             if (
                 bytes_limit is None
@@ -100,7 +105,9 @@ async def ingest(  # type: ignore[override]
             base564str = base64.b64encode(data).decode("utf-8")
             url_or_base64 = f"data:video/{file_type};base64,{base564str}"
 
-        model = vlm or self.config.vlm
+        model = vlm or self.vlm
+        if not model:
+            raise ValueError("Vision model (vlm) must be provided")
         generation_config = GenerationConfig(
             model=model,
             stream=False,
diff --git a/py/core/providers/database/prompts/vision_video.yaml b/py/core/providers/database/prompts/vision_video.yaml
index 3265b1564..1cac43fda 100644
--- a/py/core/providers/database/prompts/vision_video.yaml
+++ b/py/core/providers/database/prompts/vision_video.yaml
@@ -1,46 +1,46 @@
 video_understanding:
   template: >
     # Video-Understanding Prompt
-    
-    ## Objective  
+
+    ## Objective
     Produce a text-only reconstruction of the video that allows a reader to experience it without ever pressing play.
-    
+
     ---
-    
+
     ## Output Structure (Markdown)
-    
-    ### 1. Title  
-    A single-line, evocative summary  
-    Example:  
+
+    ### 1. Title
+    A single-line, evocative summary
+    Example:
     `Neon-Drenched Midnight Ramen: A Slow-Motion Culinary Ode`
-    
+
     ---
-    
-    ### 2. Scene-by-Scene Timeline  
+
+    ### 2. Scene-by-Scene Timeline
     | Timestamp | Visuals & Camera | Audio & Speech | On-Screen Text / Graphics | Mood |
     |-----------|------------------|----------------|---------------------------|------|
     | 00:00-00:02 | Fade-in from black; overhead establishing shot of a cramped Tokyo kitchen; warm tungsten glow | Low sizzle of pork fat, subtle city hum outside | White kanji on black: Episode 7 | Anticipatory, intimate |
     | 00:02-00:07 | Hand-held camera glides forward; steam curls like incense above a rolling boil | Water bubbles crescendo; no dialogue | None | Focused, almost meditative |
     | … | … | … | … | … |
-    
+
     ---
-    
-    ### 3. Transcript (Verbatim)  
-    - 00:10-00:15  
-      Male narrator, deep & calm, Japanese with English subtitles:  
+
+    ### 3. Transcript (Verbatim)
+    - 00:10-00:15
+      Male narrator, deep & calm, Japanese with English subtitles:
       “In an alley off Shibuya, at 2 a.m., the last customer orders a bowl of tonkotsu.”
-    
+
     ---
-    
-    ### 4. Environment & Object Inventory  
-    - Countertop: Stainless steel reflects overhead bulb; soy-sauce bottle labeled Yamasho in red calligraphy; digital timer shows 03:47.  
+
+    ### 4. Environment & Object Inventory
+    - Countertop: Stainless steel reflects overhead bulb; soy-sauce bottle labeled Yamasho in red calligraphy; digital timer shows 03:47.
     - Ingredients: Chashu cross-section reveals spiral fat marbling; green onions diced to 2 mm cubes.
-    
+
     ---
-    
-    ### 5. Style & Technical Notes  
-    - Color palette: High-contrast warm tones—dominant reds & ambers, saturation ≈ 75 %.  
-    - Motion: 60 fps slowed to 30 fps; subtle handheld micro-shakes.  
+
+    ### 5. Style & Technical Notes
+    - Color palette: High-contrast warm tones—dominant reds & ambers, saturation ≈ 75 %.
+    - Motion: 60 fps slowed to 30 fps; subtle handheld micro-shakes.
     - Soundtrack: Lo-fi hip-hop loop, 72 BPM, sparse piano chords every 8 bars.
-    
+
   input_types: {}
diff --git a/py/tests/unit/parsers/test_video_parser.py b/py/tests/unit/parsers/test_video_parser.py
index 238039834..2f19802cf 100644
--- a/py/tests/unit/parsers/test_video_parser.py
+++ b/py/tests/unit/parsers/test_video_parser.py
@@ -10,6 +10,7 @@ class DummyApp:
 
 
 class DummyConfig:
+    vlm = None
     app = DummyApp()
 
 
@@ -41,7 +42,7 @@ async def test_ingest_str_success(mock_db_provider, mock_llm_provider):
         llm_provider=mock_llm_provider,
     )
     gen = parser.ingest(
-        "http://test/video.mp4", file_type="mp4", bytes_limit=None
+        "http://test/video.mp4", extra_fields={"file_type": "mp4", "bytes_limit": 120}
     )
     result = [x async for x in gen]
     assert result == ["video description"]
@@ -55,7 +56,7 @@ async def test_ingest_bytes_success(mock_db_provider, mock_llm_provider):
         llm_provider=mock_llm_provider,
     )
     data = b"1234"
-    gen = parser.ingest(data, file_type="mp4", bytes_limit=10)
+    gen = parser.ingest(data, extra_fields={"file_type": "mp4", "bytes_limit": 10})
     result = [x async for x in gen]
     assert result == ["video description"]
 
@@ -68,7 +69,7 @@ async def test_ingest_invalid_file_type(mock_db_provider, mock_llm_provider):
         llm_provider=mock_llm_provider,
     )
     with pytest.raises(ValueError):
-        gen = parser.ingest("http://test/video.xyz", file_type="xyz")
+        gen = parser.ingest("http://test/video.xyz", extra_fields={"file_type": "xyz"})
         [x async for x in gen]
 
 
@@ -83,7 +84,7 @@ async def test_ingest_bytes_limit_exceeded(
     )
     data = b"1" * 11
     with pytest.raises(ValueError):
-        gen = parser.ingest(data, file_type="mp4", bytes_limit=10)
+        gen = parser.ingest(data, extra_fields={"file_type": "mp4", "bytes_limit": 10})
         [x async for x in gen]
 
 
@@ -97,6 +98,6 @@ async def test_ingest_llm_no_response(mock_db_provider, mock_llm_provider):
         database_provider=mock_db_provider,
         llm_provider=mock_llm_provider,
     )
-    gen = parser.ingest("http://test/video.mp4", file_type="mp4")
+    gen = parser.ingest("http://test/video.mp4", extra_fields={"file_type": "mp4"})
     with pytest.raises(ValueError):
         [x async for x in gen]