diff --git a/agentplatform/_genai/evals.py b/agentplatform/_genai/evals.py index dbbabdd1d7..9815d7c6c2 100644 --- a/agentplatform/_genai/evals.py +++ b/agentplatform/_genai/evals.py @@ -43,6 +43,56 @@ logger = logging.getLogger("agentplatform_genai.evals") +def _AgentRunConfig_from_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["sessionInput"]) is not None: + setv( + to_object, + ["session_input"], + _SessionInput_from_vertex(getv(from_object, ["sessionInput"]), to_object), + ) + + if getv(from_object, ["agentEngine"]) is not None: + setv(to_object, ["agent_engine"], getv(from_object, ["agentEngine"])) + + if getv(from_object, ["userSimulatorConfig"]) is not None: + setv( + to_object, + ["user_simulator_config"], + getv(from_object, ["userSimulatorConfig"]), + ) + + return to_object + + +def _AgentRunConfig_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["session_input"]) is not None: + setv( + to_object, + ["sessionInput"], + _SessionInput_to_vertex(getv(from_object, ["session_input"]), to_object), + ) + + if getv(from_object, ["agent_engine"]) is not None: + setv(to_object, ["agentEngine"], getv(from_object, ["agent_engine"])) + + if getv(from_object, ["user_simulator_config"]) is not None: + setv( + to_object, + ["userSimulatorConfig"], + getv(from_object, ["user_simulator_config"]), + ) + + return to_object + + def _CreateEvaluationItemParameters_to_vertex( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -137,6 +187,9 @@ def _CreateEvaluationRunParameters_to_vertex( [item for item in getv(from_object, ["analysis_configs"])], ) + if getv(from_object, ["dummy_session_input"]) is not None: + _SessionInput_to_vertex(getv(from_object, ["dummy_session_input"]), to_object) + return to_object @@ -464,7 +517,13 @@ def _EvaluationRunInferenceConfig_from_vertex( setv(to_object, ["prompt_template"], getv(from_object, ["promptTemplate"])) if getv(from_object, ["agentRunConfig"]) is not None: - setv(to_object, ["agent_run_config"], getv(from_object, ["agentRunConfig"])) + setv( + to_object, + ["agent_run_config"], + _AgentRunConfig_from_vertex( + getv(from_object, ["agentRunConfig"]), to_object + ), + ) if getv(from_object, ["agents"]) is not None: setv(to_object, ["agent_configs"], getv(from_object, ["agents"])) @@ -487,7 +546,13 @@ def _EvaluationRunInferenceConfig_to_vertex( setv(to_object, ["promptTemplate"], getv(from_object, ["prompt_template"])) if getv(from_object, ["agent_run_config"]) is not None: - setv(to_object, ["agentRunConfig"], getv(from_object, ["agent_run_config"])) + setv( + to_object, + ["agentRunConfig"], + _AgentRunConfig_to_vertex( + getv(from_object, ["agent_run_config"]), to_object + ), + ) if getv(from_object, ["agent_configs"]) is not None: setv(to_object, ["agents"], getv(from_object, ["agent_configs"])) @@ -902,6 +967,40 @@ def _RubricBasedMetricSpec_to_vertex( return to_object +def _SessionInput_from_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["userId"]) is not None: + setv(to_object, ["user_id"], getv(from_object, ["userId"])) + + if getv(from_object, ["sessionState"]) is not None: + setv(to_object, ["state"], getv(from_object, ["sessionState"])) + + if getv(from_object, ["parameters", "app_name"]) is not None: + setv(to_object, ["app_name"], getv(from_object, ["parameters", "app_name"])) + + return to_object + + +def _SessionInput_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["user_id"]) is not None: + setv(to_object, ["userId"], getv(from_object, ["user_id"])) + + if getv(from_object, ["state"]) is not None: + setv(to_object, ["sessionState"], getv(from_object, ["state"])) + + if getv(from_object, ["app_name"]) is not None: + setv(to_object, ["parameters", "app_name"], getv(from_object, ["app_name"])) + + return to_object + + def _UnifiedMetric_from_vertex( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -1174,6 +1273,10 @@ def _create_evaluation_run( ] = None, config: Optional[types.CreateEvaluationRunConfigOrDict] = None, analysis_configs: Optional[list[types.AnalysisConfigOrDict]] = None, + dummy_session_input: Optional[types.evals.SessionInputOrDict] = None, + dummy_user_simulator_config: Optional[ + types.evals.UserSimulatorConfigOrDict + ] = None, ) -> types.EvaluationRun: """ Creates an EvaluationRun. @@ -1188,6 +1291,8 @@ def _create_evaluation_run( inference_configs=inference_configs, config=config, analysis_configs=analysis_configs, + dummy_session_input=dummy_session_input, + dummy_user_simulator_config=dummy_user_simulator_config, ) request_url_dict: Optional[dict[str, str]] @@ -3321,6 +3426,10 @@ async def _create_evaluation_run( ] = None, config: Optional[types.CreateEvaluationRunConfigOrDict] = None, analysis_configs: Optional[list[types.AnalysisConfigOrDict]] = None, + dummy_session_input: Optional[types.evals.SessionInputOrDict] = None, + dummy_user_simulator_config: Optional[ + types.evals.UserSimulatorConfigOrDict + ] = None, ) -> types.EvaluationRun: """ Creates an EvaluationRun. @@ -3335,6 +3444,8 @@ async def _create_evaluation_run( inference_configs=inference_configs, config=config, analysis_configs=analysis_configs, + dummy_session_input=dummy_session_input, + dummy_user_simulator_config=dummy_user_simulator_config, ) request_url_dict: Optional[dict[str, str]] diff --git a/agentplatform/_genai/types/common.py b/agentplatform/_genai/types/common.py index 347b269a16..61ebd4485a 100644 --- a/agentplatform/_genai/types/common.py +++ b/agentplatform/_genai/types/common.py @@ -2560,13 +2560,13 @@ class AgentRunConfig(_common.BaseModel): class AgentRunConfigDict(TypedDict, total=False): """Configuration for an Agent Run.""" - session_input: Optional[evals_types.SessionInput] + session_input: Optional[evals_types.SessionInputDict] """The session input to get agent running results.""" agent_engine: Optional[str] """The resource name of the Agent Engine.""" - user_simulator_config: Optional[evals_types.UserSimulatorConfig] + user_simulator_config: Optional[evals_types.UserSimulatorConfigDict] """Used for multi-turn agent run. Contains configuration for a user simulator that uses an LLM to generate messages on behalf of the user.""" @@ -2765,6 +2765,12 @@ class _CreateEvaluationRunParameters(_common.BaseModel): analysis_configs: Optional[list[AnalysisConfig]] = Field( default=None, description="""""" ) + dummy_session_input: Optional[evals_types.SessionInput] = Field( + default=None, description="""""" + ) + dummy_user_simulator_config: Optional[evals_types.UserSimulatorConfig] = Field( + default=None, description="""""" + ) class _CreateEvaluationRunParametersDict(TypedDict, total=False): @@ -2794,6 +2800,12 @@ class _CreateEvaluationRunParametersDict(TypedDict, total=False): analysis_configs: Optional[list[AnalysisConfigDict]] """""" + dummy_session_input: Optional[evals_types.SessionInputDict] + """""" + + dummy_user_simulator_config: Optional[evals_types.UserSimulatorConfigDict] + """""" + _CreateEvaluationRunParametersOrDict = Union[ _CreateEvaluationRunParameters, _CreateEvaluationRunParametersDict diff --git a/agentplatform/_genai/types/evals.py b/agentplatform/_genai/types/evals.py index d264a58c5e..edf9a5e399 100644 --- a/agentplatform/_genai/types/evals.py +++ b/agentplatform/_genai/types/evals.py @@ -36,6 +36,85 @@ class Importance(_common.CaseInSensitiveEnum): """Low importance.""" +class SessionInput(_common.BaseModel): + """This field is experimental and may change in future versions. + + Input to initialize a session and run an agent, used for agent evaluation. + """ + + user_id: Optional[str] = Field(default=None, description="""The user id.""") + state: Optional[dict[str, str]] = Field( + default=None, description="""The state of the session.""" + ) + app_name: Optional[str] = Field( + default=None, + description="""The name of the app, used for local ADK agent run Runner and Session.""", + ) + + +class SessionInputDict(TypedDict, total=False): + """This field is experimental and may change in future versions. + + Input to initialize a session and run an agent, used for agent evaluation. + """ + + user_id: Optional[str] + """The user id.""" + + state: Optional[dict[str, str]] + """The state of the session.""" + + app_name: Optional[str] + """The name of the app, used for local ADK agent run Runner and Session.""" + + +SessionInputOrDict = Union[SessionInput, SessionInputDict] + + +class UserSimulatorConfig(_common.BaseModel): + """Configuration for a user simulator. + + Uses an LLM to generate multi-turn messages that simulate a user. + """ + + model_name: Optional[str] = Field( + default=None, + description="""The model name to get next user message for multi-turn agent run.""", + ) + model_configuration: Optional[genai_types.GenerateContentConfig] = Field( + default=None, description="""The configuration for the model.""" + ) + max_turn: Optional[int] = Field( + default=None, + description="""Maximum number of invocations allowed by the multi-turn agent + running. This property allows us to stop a run-off conversation + where the agent and the user simulator get into a never ending loop. + The initial fixed prompt is also counted as an invocation.""", + ) + + +class UserSimulatorConfigDict(TypedDict, total=False): + """Configuration for a user simulator. + + Uses an LLM to generate multi-turn messages that simulate a user. + """ + + model_name: Optional[str] + """The model name to get next user message for multi-turn agent run.""" + + model_configuration: Optional[genai_types.GenerateContentConfigDict] + """The configuration for the model.""" + + max_turn: Optional[int] + """Maximum number of invocations allowed by the multi-turn agent + running. This property allows us to stop a run-off conversation + where the agent and the user simulator get into a never ending loop. + The initial fixed prompt is also counted as an invocation.""" + + +UserSimulatorConfigOrDict = Union[UserSimulatorConfig, UserSimulatorConfigDict] + + class AgentConfig(_common.BaseModel): """Represents configuration for an Agent.""" @@ -458,41 +537,6 @@ class AgentInfoDict(TypedDict, total=False): AgentInfoOrDict = Union[AgentInfo, AgentInfoDict] -class SessionInput(_common.BaseModel): - """This field is experimental and may change in future versions. - - Input to initialize a session and run an agent, used for agent evaluation. - """ - - user_id: Optional[str] = Field(default=None, description="""The user id.""") - state: Optional[dict[str, str]] = Field( - default=None, description="""The state of the session.""" - ) - app_name: Optional[str] = Field( - default=None, - description="""The name of the app, used for local ADK agent run Runner and Session.""", - ) - - -class SessionInputDict(TypedDict, total=False): - """This field is experimental and may change in future versions. - - Input to initialize a session and run an agent, used for agent evaluation. - """ - - user_id: Optional[str] - """The user id.""" - - state: Optional[dict[str, str]] - """The state of the session.""" - - app_name: Optional[str] - """The name of the app, used for local ADK agent run Runner and Session.""" - - -SessionInputOrDict = Union[SessionInput, SessionInputDict] - - class UserScenario(_common.BaseModel): """User scenario to help simulate multi-turn agent run results.""" @@ -588,50 +632,6 @@ class UserScenarioGenerationConfigDict(TypedDict, total=False): ] -class UserSimulatorConfig(_common.BaseModel): - """Configuration for a user simulator. - - Uses an LLM to generate multi-turn messages that simulate a user. - """ - - model_name: Optional[str] = Field( - default=None, - description="""The model name to get next user message for multi-turn agent run.""", - ) - model_configuration: Optional[genai_types.GenerateContentConfig] = Field( - default=None, description="""The configuration for the model.""" - ) - max_turn: Optional[int] = Field( - default=None, - description="""Maximum number of invocations allowed by the multi-turn agent - running. This property allows us to stop a run-off conversation - where the agent and the user simulator get into a never ending loop. - The initial fixed prompt is also counted as an invocation.""", - ) - - -class UserSimulatorConfigDict(TypedDict, total=False): - """Configuration for a user simulator. - - Uses an LLM to generate multi-turn messages that simulate a user. - """ - - model_name: Optional[str] - """The model name to get next user message for multi-turn agent run.""" - - model_configuration: Optional[genai_types.GenerateContentConfigDict] - """The configuration for the model.""" - - max_turn: Optional[int] - """Maximum number of invocations allowed by the multi-turn agent - running. This property allows us to stop a run-off conversation - where the agent and the user simulator get into a never ending loop. - The initial fixed prompt is also counted as an invocation.""" - - -UserSimulatorConfigOrDict = Union[UserSimulatorConfig, UserSimulatorConfigDict] - - class Event(_common.BaseModel): """Represents an event in a conversation between agents and users.