Viewing docs for Google Cloud v9.24.0
published on Friday, May 22, 2026 by Pulumi
published on Friday, May 22, 2026 by Pulumi
Viewing docs for Google Cloud v9.24.0
published on Friday, May 22, 2026 by Pulumi
published on Friday, May 22, 2026 by Pulumi
Customer Engagement Suite Evaluation
Warning: This resource is in beta, and should be used with the terraform-provider-google-beta provider. See Provider Versions for more details on beta resources.
To get more information about Evaluation, see:
Example Usage
Ces Evaluation Basic
import * as pulumi from "@pulumi/pulumi";
import * as gcp from "@pulumi/gcp";
const app = new gcp.ces.App("app", {
appId: "app-id",
location: "us",
displayName: "my-app",
languageSettings: {
defaultLanguageCode: "en-US",
},
timeZoneSettings: {
timeZone: "America/Los_Angeles",
},
});
const cesEvaluationBasic = new gcp.ces.Evaluation("ces_evaluation_basic", {
evaluationId: "eval-basic",
displayName: "my-evaluation-basic",
location: "us",
app: app.appId,
});
import pulumi
import pulumi_gcp as gcp
app = gcp.ces.App("app",
app_id="app-id",
location="us",
display_name="my-app",
language_settings={
"default_language_code": "en-US",
},
time_zone_settings={
"time_zone": "America/Los_Angeles",
})
ces_evaluation_basic = gcp.ces.Evaluation("ces_evaluation_basic",
evaluation_id="eval-basic",
display_name="my-evaluation-basic",
location="us",
app=app.app_id)
package main
import (
"github.com/pulumi/pulumi-gcp/sdk/v9/go/gcp/ces"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
app, err := ces.NewApp(ctx, "app", &ces.AppArgs{
AppId: pulumi.String("app-id"),
Location: pulumi.String("us"),
DisplayName: pulumi.String("my-app"),
LanguageSettings: &ces.AppLanguageSettingsArgs{
DefaultLanguageCode: pulumi.String("en-US"),
},
TimeZoneSettings: &ces.AppTimeZoneSettingsArgs{
TimeZone: pulumi.String("America/Los_Angeles"),
},
})
if err != nil {
return err
}
_, err = ces.NewEvaluation(ctx, "ces_evaluation_basic", &ces.EvaluationArgs{
EvaluationId: pulumi.String("eval-basic"),
DisplayName: pulumi.String("my-evaluation-basic"),
Location: pulumi.String("us"),
App: app.AppId,
})
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Gcp = Pulumi.Gcp;
return await Deployment.RunAsync(() =>
{
var app = new Gcp.Ces.App("app", new()
{
AppId = "app-id",
Location = "us",
DisplayName = "my-app",
LanguageSettings = new Gcp.Ces.Inputs.AppLanguageSettingsArgs
{
DefaultLanguageCode = "en-US",
},
TimeZoneSettings = new Gcp.Ces.Inputs.AppTimeZoneSettingsArgs
{
TimeZone = "America/Los_Angeles",
},
});
var cesEvaluationBasic = new Gcp.Ces.Evaluation("ces_evaluation_basic", new()
{
EvaluationId = "eval-basic",
DisplayName = "my-evaluation-basic",
Location = "us",
App = app.AppId,
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.gcp.ces.App;
import com.pulumi.gcp.ces.AppArgs;
import com.pulumi.gcp.ces.inputs.AppLanguageSettingsArgs;
import com.pulumi.gcp.ces.inputs.AppTimeZoneSettingsArgs;
import com.pulumi.gcp.ces.Evaluation;
import com.pulumi.gcp.ces.EvaluationArgs;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var app = new App("app", AppArgs.builder()
.appId("app-id")
.location("us")
.displayName("my-app")
.languageSettings(AppLanguageSettingsArgs.builder()
.defaultLanguageCode("en-US")
.build())
.timeZoneSettings(AppTimeZoneSettingsArgs.builder()
.timeZone("America/Los_Angeles")
.build())
.build());
var cesEvaluationBasic = new Evaluation("cesEvaluationBasic", EvaluationArgs.builder()
.evaluationId("eval-basic")
.displayName("my-evaluation-basic")
.location("us")
.app(app.appId())
.build());
}
}
resources:
app:
type: gcp:ces:App
properties:
appId: app-id
location: us
displayName: my-app
languageSettings:
defaultLanguageCode: en-US
timeZoneSettings:
timeZone: America/Los_Angeles
cesEvaluationBasic:
type: gcp:ces:Evaluation
name: ces_evaluation_basic
properties:
evaluationId: eval-basic
displayName: my-evaluation-basic
location: us
app: ${app.appId}
Example coming soon!
Ces Evaluation Full
import * as pulumi from "@pulumi/pulumi";
import * as gcp from "@pulumi/gcp";
const app = new gcp.ces.App("app", {
appId: "app-id-full",
location: "us",
displayName: "my-app-full",
languageSettings: {
defaultLanguageCode: "en-US",
},
timeZoneSettings: {
timeZone: "America/Los_Angeles",
},
});
const tool = new gcp.ces.Tool("tool", {
location: "us",
app: app.appId,
toolId: "tool-id-full",
executionType: "SYNCHRONOUS",
pythonFunction: {
name: "example_function",
pythonCode: "def example_function() -> int: return 0",
},
});
const cesEvaluationFull = new gcp.ces.Evaluation("ces_evaluation_full", {
evaluationId: "evaluation-id-full",
displayName: "my-evaluation-full",
location: "us",
app: app.appId,
description: "Full evaluation for testing",
tags: [
"test",
"full",
],
golden: {
evaluationExpectations: [pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/evaluationExpectations/dummy-exp`],
turns: [{
steps: [
{
userInput: {
text: "Hello",
willContinue: true,
},
},
{
userInput: {
variables: {
key: "value",
},
},
},
{
expectation: {
note: "Expect tool call",
toolCall: {
id: "tool-call-id",
tool: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}`,
args: {
param: "value",
},
},
},
},
{
expectation: {
note: "Expect agent response",
agentResponse: {
role: "agent",
chunks: [
{
updatedVariables: {
key: "value",
},
},
{
agentTransfer: {
targetAgent: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/agents/dummy-agent`,
},
},
{
toolCall: {
id: "tool-call-id-3",
tool: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}`,
},
},
{
toolResponse: {
id: "tool-call-id-3",
response: {
result: "success",
},
tool: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}`,
},
},
],
},
},
},
{
expectation: {
note: "Expect toolset tool call",
toolCall: {
id: "tool-call-id-2",
toolsetTool: {
toolset: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/toolsets/dummy-toolset`,
toolId: "dummy-tool",
},
},
},
},
{
agentTransfer: {
targetAgent: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/agents/dummy-agent`,
},
},
{
expectation: {
agentTransfer: {
targetAgent: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/agents/dummy-agent`,
displayName: "dummy-agent",
},
},
},
{
expectation: {
note: "Expect mock tool response",
mockToolResponse: {
id: "tool-call-id-4",
response: {
result: "mocked",
},
tool: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}`,
},
},
},
{
expectation: {
note: "Expect tool response",
toolResponse: {
toolsetTool: {
toolset: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/toolsets/dummy-toolset`,
toolId: "dummy-tool",
},
},
},
},
{
expectation: {
note: "Expect updated variables",
updatedVariables: {
notes: "Some notes",
},
},
},
{
userInput: {
toolResponses: {
toolResponses: [{
id: "tool-call-id-5",
response: {
result: "user-provided",
},
tool: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}`,
}],
},
},
},
{
userInput: {
event: {
event: "my-event",
},
},
},
{
userInput: {
dtmf: "1234",
},
},
{
userInput: {
audio: "c29tZSBhdWRpbyBkYXRh",
},
},
{
userInput: {
blob: {
mimeType: "text/plain",
data: "c29tZSBibG9iIGRhdGE=",
},
},
},
{
userInput: {
image: {
mimeType: "image/png",
data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=",
},
},
},
{
userInput: {
toolResponses: {
toolResponses: [{
id: "tool-call-id",
response: {
result: "success",
},
tool: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}`,
}],
},
},
},
{
expectation: {
note: "Expect tool response",
toolResponse: {
id: "tool-call-id",
response: {
result: "success",
},
tool: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}`,
},
},
},
{
expectation: {
note: "Expect agent response with chunks",
agentResponse: {
role: "agent",
chunks: [
{
text: "Hello again",
},
{
text: "Hello again transcript",
},
{
blob: {
mimeType: "text/plain",
data: "c29tZSBibG9iIGRhdGE=",
},
},
{
image: {
mimeType: "image/png",
data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=",
},
},
{
toolCall: {
id: "tool-call-id-3",
tool: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}`,
args: {
param: "value",
},
},
},
],
},
},
},
],
}],
},
});
import pulumi
import pulumi_gcp as gcp
app = gcp.ces.App("app",
app_id="app-id-full",
location="us",
display_name="my-app-full",
language_settings={
"default_language_code": "en-US",
},
time_zone_settings={
"time_zone": "America/Los_Angeles",
})
tool = gcp.ces.Tool("tool",
location="us",
app=app.app_id,
tool_id="tool-id-full",
execution_type="SYNCHRONOUS",
python_function={
"name": "example_function",
"python_code": "def example_function() -> int: return 0",
})
ces_evaluation_full = gcp.ces.Evaluation("ces_evaluation_full",
evaluation_id="evaluation-id-full",
display_name="my-evaluation-full",
location="us",
app=app.app_id,
description="Full evaluation for testing",
tags=[
"test",
"full",
],
golden={
"evaluation_expectations": [pulumi.Output.all(
project=app.project,
app_id=app.app_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/evaluationExpectations/dummy-exp")
],
"turns": [{
"steps": [
{
"user_input": {
"text": "Hello",
"will_continue": True,
},
},
{
"user_input": {
"variables": {
"key": "value",
},
},
},
{
"expectation": {
"note": "Expect tool call",
"tool_call": {
"id": "tool-call-id",
"tool": pulumi.Output.all(
project=app.project,
app_id=app.app_id,
tool_id=tool.tool_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/tools/{resolved_outputs['tool_id']}")
,
"args": {
"param": "value",
},
},
},
},
{
"expectation": {
"note": "Expect agent response",
"agent_response": {
"role": "agent",
"chunks": [
{
"updated_variables": {
"key": "value",
},
},
{
"agent_transfer": {
"target_agent": pulumi.Output.all(
project=app.project,
app_id=app.app_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/agents/dummy-agent")
,
},
},
{
"tool_call": {
"id": "tool-call-id-3",
"tool": pulumi.Output.all(
project=app.project,
app_id=app.app_id,
tool_id=tool.tool_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/tools/{resolved_outputs['tool_id']}")
,
},
},
{
"tool_response": {
"id": "tool-call-id-3",
"response": {
"result": "success",
},
"tool": pulumi.Output.all(
project=app.project,
app_id=app.app_id,
tool_id=tool.tool_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/tools/{resolved_outputs['tool_id']}")
,
},
},
],
},
},
},
{
"expectation": {
"note": "Expect toolset tool call",
"tool_call": {
"id": "tool-call-id-2",
"toolset_tool": {
"toolset": pulumi.Output.all(
project=app.project,
app_id=app.app_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/toolsets/dummy-toolset")
,
"tool_id": "dummy-tool",
},
},
},
},
{
"agent_transfer": {
"target_agent": pulumi.Output.all(
project=app.project,
app_id=app.app_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/agents/dummy-agent")
,
},
},
{
"expectation": {
"agent_transfer": {
"target_agent": pulumi.Output.all(
project=app.project,
app_id=app.app_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/agents/dummy-agent")
,
"display_name": "dummy-agent",
},
},
},
{
"expectation": {
"note": "Expect mock tool response",
"mock_tool_response": {
"id": "tool-call-id-4",
"response": {
"result": "mocked",
},
"tool": pulumi.Output.all(
project=app.project,
app_id=app.app_id,
tool_id=tool.tool_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/tools/{resolved_outputs['tool_id']}")
,
},
},
},
{
"expectation": {
"note": "Expect tool response",
"tool_response": {
"toolset_tool": {
"toolset": pulumi.Output.all(
project=app.project,
app_id=app.app_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/toolsets/dummy-toolset")
,
"tool_id": "dummy-tool",
},
},
},
},
{
"expectation": {
"note": "Expect updated variables",
"updated_variables": {
"notes": "Some notes",
},
},
},
{
"user_input": {
"tool_responses": {
"tool_responses": [{
"id": "tool-call-id-5",
"response": {
"result": "user-provided",
},
"tool": pulumi.Output.all(
project=app.project,
app_id=app.app_id,
tool_id=tool.tool_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/tools/{resolved_outputs['tool_id']}")
,
}],
},
},
},
{
"user_input": {
"event": {
"event": "my-event",
},
},
},
{
"user_input": {
"dtmf": "1234",
},
},
{
"user_input": {
"audio": "c29tZSBhdWRpbyBkYXRh",
},
},
{
"user_input": {
"blob": {
"mime_type": "text/plain",
"data": "c29tZSBibG9iIGRhdGE=",
},
},
},
{
"user_input": {
"image": {
"mime_type": "image/png",
"data": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=",
},
},
},
{
"user_input": {
"tool_responses": {
"tool_responses": [{
"id": "tool-call-id",
"response": {
"result": "success",
},
"tool": pulumi.Output.all(
project=app.project,
app_id=app.app_id,
tool_id=tool.tool_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/tools/{resolved_outputs['tool_id']}")
,
}],
},
},
},
{
"expectation": {
"note": "Expect tool response",
"tool_response": {
"id": "tool-call-id",
"response": {
"result": "success",
},
"tool": pulumi.Output.all(
project=app.project,
app_id=app.app_id,
tool_id=tool.tool_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/tools/{resolved_outputs['tool_id']}")
,
},
},
},
{
"expectation": {
"note": "Expect agent response with chunks",
"agent_response": {
"role": "agent",
"chunks": [
{
"text": "Hello again",
},
{
"text": "Hello again transcript",
},
{
"blob": {
"mime_type": "text/plain",
"data": "c29tZSBibG9iIGRhdGE=",
},
},
{
"image": {
"mime_type": "image/png",
"data": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=",
},
},
{
"tool_call": {
"id": "tool-call-id-3",
"tool": pulumi.Output.all(
project=app.project,
app_id=app.app_id,
tool_id=tool.tool_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/tools/{resolved_outputs['tool_id']}")
,
"args": {
"param": "value",
},
},
},
],
},
},
},
],
}],
})
package main
import (
"fmt"
"github.com/pulumi/pulumi-gcp/sdk/v9/go/gcp/ces"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
app, err := ces.NewApp(ctx, "app", &ces.AppArgs{
AppId: pulumi.String("app-id-full"),
Location: pulumi.String("us"),
DisplayName: pulumi.String("my-app-full"),
LanguageSettings: &ces.AppLanguageSettingsArgs{
DefaultLanguageCode: pulumi.String("en-US"),
},
TimeZoneSettings: &ces.AppTimeZoneSettingsArgs{
TimeZone: pulumi.String("America/Los_Angeles"),
},
})
if err != nil {
return err
}
tool, err := ces.NewTool(ctx, "tool", &ces.ToolArgs{
Location: pulumi.String("us"),
App: app.AppId,
ToolId: pulumi.String("tool-id-full"),
ExecutionType: pulumi.String("SYNCHRONOUS"),
PythonFunction: &ces.ToolPythonFunctionArgs{
Name: pulumi.String("example_function"),
PythonCode: pulumi.String("def example_function() -> int: return 0"),
},
})
if err != nil {
return err
}
_, err = ces.NewEvaluation(ctx, "ces_evaluation_full", &ces.EvaluationArgs{
EvaluationId: pulumi.String("evaluation-id-full"),
DisplayName: pulumi.String("my-evaluation-full"),
Location: pulumi.String("us"),
App: app.AppId,
Description: pulumi.String("Full evaluation for testing"),
Tags: pulumi.StringArray{
pulumi.String("test"),
pulumi.String("full"),
},
Golden: &ces.EvaluationGoldenArgs{
EvaluationExpectations: pulumi.StringArray{
pulumi.All(app.Project, app.AppId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/evaluationExpectations/dummy-exp", project, appId), nil
}).(pulumi.StringOutput),
},
Turns: ces.EvaluationGoldenTurnArray{
&ces.EvaluationGoldenTurnArgs{
Steps: ces.EvaluationGoldenTurnStepArray{
&ces.EvaluationGoldenTurnStepArgs{
UserInput: &ces.EvaluationGoldenTurnStepUserInputArgs{
Text: pulumi.String("Hello"),
WillContinue: pulumi.Bool(true),
},
},
&ces.EvaluationGoldenTurnStepArgs{
UserInput: &ces.EvaluationGoldenTurnStepUserInputArgs{
Variables: pulumi.StringMap{
"key": pulumi.String("value"),
},
},
},
&ces.EvaluationGoldenTurnStepArgs{
Expectation: &ces.EvaluationGoldenTurnStepExpectationArgs{
Note: pulumi.String("Expect tool call"),
ToolCall: &ces.EvaluationGoldenTurnStepExpectationToolCallArgs{
Id: pulumi.String("tool-call-id"),
Tool: pulumi.All(app.Project, app.AppId, tool.ToolId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
toolId := _args[2].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/tools/%v", project, appId, toolId), nil
}).(pulumi.StringOutput),
Args: pulumi.StringMap{
"param": pulumi.String("value"),
},
},
},
},
&ces.EvaluationGoldenTurnStepArgs{
Expectation: &ces.EvaluationGoldenTurnStepExpectationArgs{
Note: pulumi.String("Expect agent response"),
AgentResponse: &ces.EvaluationGoldenTurnStepExpectationAgentResponseArgs{
Role: pulumi.String("agent"),
Chunks: ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkArray{
&ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs{
UpdatedVariables: pulumi.StringMap{
"key": pulumi.String("value"),
},
},
&ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs{
AgentTransfer: &ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkAgentTransferArgs{
TargetAgent: pulumi.All(app.Project, app.AppId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/agents/dummy-agent", project, appId), nil
}).(pulumi.StringOutput),
},
},
&ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs{
ToolCall: &ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallArgs{
Id: pulumi.String("tool-call-id-3"),
Tool: pulumi.All(app.Project, app.AppId, tool.ToolId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
toolId := _args[2].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/tools/%v", project, appId, toolId), nil
}).(pulumi.StringOutput),
},
},
&ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs{
ToolResponse: &ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseArgs{
Id: pulumi.String("tool-call-id-3"),
Response: pulumi.StringMap{
"result": pulumi.String("success"),
},
Tool: pulumi.All(app.Project, app.AppId, tool.ToolId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
toolId := _args[2].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/tools/%v", project, appId, toolId), nil
}).(pulumi.StringOutput),
},
},
},
},
},
},
&ces.EvaluationGoldenTurnStepArgs{
Expectation: &ces.EvaluationGoldenTurnStepExpectationArgs{
Note: pulumi.String("Expect toolset tool call"),
ToolCall: &ces.EvaluationGoldenTurnStepExpectationToolCallArgs{
Id: pulumi.String("tool-call-id-2"),
ToolsetTool: &ces.EvaluationGoldenTurnStepExpectationToolCallToolsetToolArgs{
Toolset: pulumi.All(app.Project, app.AppId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/toolsets/dummy-toolset", project, appId), nil
}).(pulumi.StringOutput),
ToolId: pulumi.String("dummy-tool"),
},
},
},
},
&ces.EvaluationGoldenTurnStepArgs{
AgentTransfer: &ces.EvaluationGoldenTurnStepAgentTransferArgs{
TargetAgent: pulumi.All(app.Project, app.AppId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/agents/dummy-agent", project, appId), nil
}).(pulumi.StringOutput),
},
},
&ces.EvaluationGoldenTurnStepArgs{
Expectation: &ces.EvaluationGoldenTurnStepExpectationArgs{
AgentTransfer: &ces.EvaluationGoldenTurnStepExpectationAgentTransferArgs{
TargetAgent: pulumi.All(app.Project, app.AppId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/agents/dummy-agent", project, appId), nil
}).(pulumi.StringOutput),
DisplayName: pulumi.String("dummy-agent"),
},
},
},
&ces.EvaluationGoldenTurnStepArgs{
Expectation: &ces.EvaluationGoldenTurnStepExpectationArgs{
Note: pulumi.String("Expect mock tool response"),
MockToolResponse: &ces.EvaluationGoldenTurnStepExpectationMockToolResponseArgs{
Id: pulumi.String("tool-call-id-4"),
Response: pulumi.StringMap{
"result": pulumi.String("mocked"),
},
Tool: pulumi.All(app.Project, app.AppId, tool.ToolId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
toolId := _args[2].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/tools/%v", project, appId, toolId), nil
}).(pulumi.StringOutput),
},
},
},
&ces.EvaluationGoldenTurnStepArgs{
Expectation: &ces.EvaluationGoldenTurnStepExpectationArgs{
Note: pulumi.String("Expect tool response"),
ToolResponse: &ces.EvaluationGoldenTurnStepExpectationToolResponseArgs{
ToolsetTool: &ces.EvaluationGoldenTurnStepExpectationToolResponseToolsetToolArgs{
Toolset: pulumi.All(app.Project, app.AppId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/toolsets/dummy-toolset", project, appId), nil
}).(pulumi.StringOutput),
ToolId: pulumi.String("dummy-tool"),
},
},
},
},
&ces.EvaluationGoldenTurnStepArgs{
Expectation: &ces.EvaluationGoldenTurnStepExpectationArgs{
Note: pulumi.String("Expect updated variables"),
UpdatedVariables: &ces.EvaluationGoldenTurnStepExpectationUpdatedVariablesArgs{
Notes: pulumi.String("Some notes"),
},
},
},
&ces.EvaluationGoldenTurnStepArgs{
UserInput: &ces.EvaluationGoldenTurnStepUserInputArgs{
ToolResponses: &ces.EvaluationGoldenTurnStepUserInputToolResponsesArgs{
ToolResponses: ces.EvaluationGoldenTurnStepUserInputToolResponsesToolResponseArray{
&ces.EvaluationGoldenTurnStepUserInputToolResponsesToolResponseArgs{
Id: pulumi.String("tool-call-id-5"),
Response: pulumi.StringMap{
"result": pulumi.String("user-provided"),
},
Tool: pulumi.All(app.Project, app.AppId, tool.ToolId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
toolId := _args[2].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/tools/%v", project, appId, toolId), nil
}).(pulumi.StringOutput),
},
},
},
},
},
&ces.EvaluationGoldenTurnStepArgs{
UserInput: &ces.EvaluationGoldenTurnStepUserInputArgs{
Event: &ces.EvaluationGoldenTurnStepUserInputEventArgs{
Event: pulumi.String("my-event"),
},
},
},
&ces.EvaluationGoldenTurnStepArgs{
UserInput: &ces.EvaluationGoldenTurnStepUserInputArgs{
Dtmf: pulumi.String("1234"),
},
},
&ces.EvaluationGoldenTurnStepArgs{
UserInput: &ces.EvaluationGoldenTurnStepUserInputArgs{
Audio: pulumi.String("c29tZSBhdWRpbyBkYXRh"),
},
},
&ces.EvaluationGoldenTurnStepArgs{
UserInput: &ces.EvaluationGoldenTurnStepUserInputArgs{
Blob: &ces.EvaluationGoldenTurnStepUserInputBlobArgs{
MimeType: pulumi.String("text/plain"),
Data: pulumi.String("c29tZSBibG9iIGRhdGE="),
},
},
},
&ces.EvaluationGoldenTurnStepArgs{
UserInput: &ces.EvaluationGoldenTurnStepUserInputArgs{
Image: &ces.EvaluationGoldenTurnStepUserInputImageArgs{
MimeType: pulumi.String("image/png"),
Data: pulumi.String("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="),
},
},
},
&ces.EvaluationGoldenTurnStepArgs{
UserInput: &ces.EvaluationGoldenTurnStepUserInputArgs{
ToolResponses: &ces.EvaluationGoldenTurnStepUserInputToolResponsesArgs{
ToolResponses: ces.EvaluationGoldenTurnStepUserInputToolResponsesToolResponseArray{
&ces.EvaluationGoldenTurnStepUserInputToolResponsesToolResponseArgs{
Id: pulumi.String("tool-call-id"),
Response: pulumi.StringMap{
"result": pulumi.String("success"),
},
Tool: pulumi.All(app.Project, app.AppId, tool.ToolId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
toolId := _args[2].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/tools/%v", project, appId, toolId), nil
}).(pulumi.StringOutput),
},
},
},
},
},
&ces.EvaluationGoldenTurnStepArgs{
Expectation: &ces.EvaluationGoldenTurnStepExpectationArgs{
Note: pulumi.String("Expect tool response"),
ToolResponse: &ces.EvaluationGoldenTurnStepExpectationToolResponseArgs{
Id: pulumi.String("tool-call-id"),
Response: pulumi.StringMap{
"result": pulumi.String("success"),
},
Tool: pulumi.All(app.Project, app.AppId, tool.ToolId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
toolId := _args[2].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/tools/%v", project, appId, toolId), nil
}).(pulumi.StringOutput),
},
},
},
&ces.EvaluationGoldenTurnStepArgs{
Expectation: &ces.EvaluationGoldenTurnStepExpectationArgs{
Note: pulumi.String("Expect agent response with chunks"),
AgentResponse: &ces.EvaluationGoldenTurnStepExpectationAgentResponseArgs{
Role: pulumi.String("agent"),
Chunks: ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkArray{
&ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs{
Text: pulumi.String("Hello again"),
},
&ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs{
Text: pulumi.String("Hello again transcript"),
},
&ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs{
Blob: &ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkBlobArgs{
MimeType: pulumi.String("text/plain"),
Data: pulumi.String("c29tZSBibG9iIGRhdGE="),
},
},
&ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs{
Image: &ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkImageArgs{
MimeType: pulumi.String("image/png"),
Data: pulumi.String("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="),
},
},
&ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs{
ToolCall: &ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallArgs{
Id: pulumi.String("tool-call-id-3"),
Tool: pulumi.All(app.Project, app.AppId, tool.ToolId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
toolId := _args[2].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/tools/%v", project, appId, toolId), nil
}).(pulumi.StringOutput),
Args: pulumi.StringMap{
"param": pulumi.String("value"),
},
},
},
},
},
},
},
},
},
},
},
})
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Gcp = Pulumi.Gcp;
return await Deployment.RunAsync(() =>
{
var app = new Gcp.Ces.App("app", new()
{
AppId = "app-id-full",
Location = "us",
DisplayName = "my-app-full",
LanguageSettings = new Gcp.Ces.Inputs.AppLanguageSettingsArgs
{
DefaultLanguageCode = "en-US",
},
TimeZoneSettings = new Gcp.Ces.Inputs.AppTimeZoneSettingsArgs
{
TimeZone = "America/Los_Angeles",
},
});
var tool = new Gcp.Ces.Tool("tool", new()
{
Location = "us",
App = app.AppId,
ToolId = "tool-id-full",
ExecutionType = "SYNCHRONOUS",
PythonFunction = new Gcp.Ces.Inputs.ToolPythonFunctionArgs
{
Name = "example_function",
PythonCode = "def example_function() -> int: return 0",
},
});
var cesEvaluationFull = new Gcp.Ces.Evaluation("ces_evaluation_full", new()
{
EvaluationId = "evaluation-id-full",
DisplayName = "my-evaluation-full",
Location = "us",
App = app.AppId,
Description = "Full evaluation for testing",
Tags = new[]
{
"test",
"full",
},
Golden = new Gcp.Ces.Inputs.EvaluationGoldenArgs
{
EvaluationExpectations = new[]
{
Output.Tuple(app.Project, app.AppId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
return $"projects/{project}/locations/us/apps/{appId}/evaluationExpectations/dummy-exp";
}),
},
Turns = new[]
{
new Gcp.Ces.Inputs.EvaluationGoldenTurnArgs
{
Steps = new[]
{
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
UserInput = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputArgs
{
Text = "Hello",
WillContinue = true,
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
UserInput = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputArgs
{
Variables =
{
{ "key", "value" },
},
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
Expectation = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationArgs
{
Note = "Expect tool call",
ToolCall = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationToolCallArgs
{
Id = "tool-call-id",
Tool = Output.Tuple(app.Project, app.AppId, tool.ToolId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
var toolId = values.Item3;
return $"projects/{project}/locations/us/apps/{appId}/tools/{toolId}";
}),
Args =
{
{ "param", "value" },
},
},
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
Expectation = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationArgs
{
Note = "Expect agent response",
AgentResponse = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseArgs
{
Role = "agent",
Chunks = new[]
{
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs
{
UpdatedVariables =
{
{ "key", "value" },
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs
{
AgentTransfer = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkAgentTransferArgs
{
TargetAgent = Output.Tuple(app.Project, app.AppId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
return $"projects/{project}/locations/us/apps/{appId}/agents/dummy-agent";
}),
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs
{
ToolCall = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallArgs
{
Id = "tool-call-id-3",
Tool = Output.Tuple(app.Project, app.AppId, tool.ToolId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
var toolId = values.Item3;
return $"projects/{project}/locations/us/apps/{appId}/tools/{toolId}";
}),
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs
{
ToolResponse = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseArgs
{
Id = "tool-call-id-3",
Response =
{
{ "result", "success" },
},
Tool = Output.Tuple(app.Project, app.AppId, tool.ToolId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
var toolId = values.Item3;
return $"projects/{project}/locations/us/apps/{appId}/tools/{toolId}";
}),
},
},
},
},
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
Expectation = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationArgs
{
Note = "Expect toolset tool call",
ToolCall = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationToolCallArgs
{
Id = "tool-call-id-2",
ToolsetTool = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationToolCallToolsetToolArgs
{
Toolset = Output.Tuple(app.Project, app.AppId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
return $"projects/{project}/locations/us/apps/{appId}/toolsets/dummy-toolset";
}),
ToolId = "dummy-tool",
},
},
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
AgentTransfer = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepAgentTransferArgs
{
TargetAgent = Output.Tuple(app.Project, app.AppId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
return $"projects/{project}/locations/us/apps/{appId}/agents/dummy-agent";
}),
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
Expectation = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationArgs
{
AgentTransfer = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentTransferArgs
{
TargetAgent = Output.Tuple(app.Project, app.AppId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
return $"projects/{project}/locations/us/apps/{appId}/agents/dummy-agent";
}),
DisplayName = "dummy-agent",
},
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
Expectation = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationArgs
{
Note = "Expect mock tool response",
MockToolResponse = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationMockToolResponseArgs
{
Id = "tool-call-id-4",
Response =
{
{ "result", "mocked" },
},
Tool = Output.Tuple(app.Project, app.AppId, tool.ToolId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
var toolId = values.Item3;
return $"projects/{project}/locations/us/apps/{appId}/tools/{toolId}";
}),
},
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
Expectation = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationArgs
{
Note = "Expect tool response",
ToolResponse = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationToolResponseArgs
{
ToolsetTool = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationToolResponseToolsetToolArgs
{
Toolset = Output.Tuple(app.Project, app.AppId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
return $"projects/{project}/locations/us/apps/{appId}/toolsets/dummy-toolset";
}),
ToolId = "dummy-tool",
},
},
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
Expectation = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationArgs
{
Note = "Expect updated variables",
UpdatedVariables = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationUpdatedVariablesArgs
{
Notes = "Some notes",
},
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
UserInput = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputArgs
{
ToolResponses = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputToolResponsesArgs
{
ToolResponses = new[]
{
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputToolResponsesToolResponseArgs
{
Id = "tool-call-id-5",
Response =
{
{ "result", "user-provided" },
},
Tool = Output.Tuple(app.Project, app.AppId, tool.ToolId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
var toolId = values.Item3;
return $"projects/{project}/locations/us/apps/{appId}/tools/{toolId}";
}),
},
},
},
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
UserInput = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputArgs
{
Event = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputEventArgs
{
Event = "my-event",
},
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
UserInput = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputArgs
{
Dtmf = "1234",
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
UserInput = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputArgs
{
Audio = "c29tZSBhdWRpbyBkYXRh",
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
UserInput = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputArgs
{
Blob = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputBlobArgs
{
MimeType = "text/plain",
Data = "c29tZSBibG9iIGRhdGE=",
},
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
UserInput = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputArgs
{
Image = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputImageArgs
{
MimeType = "image/png",
Data = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=",
},
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
UserInput = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputArgs
{
ToolResponses = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputToolResponsesArgs
{
ToolResponses = new[]
{
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputToolResponsesToolResponseArgs
{
Id = "tool-call-id",
Response =
{
{ "result", "success" },
},
Tool = Output.Tuple(app.Project, app.AppId, tool.ToolId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
var toolId = values.Item3;
return $"projects/{project}/locations/us/apps/{appId}/tools/{toolId}";
}),
},
},
},
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
Expectation = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationArgs
{
Note = "Expect tool response",
ToolResponse = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationToolResponseArgs
{
Id = "tool-call-id",
Response =
{
{ "result", "success" },
},
Tool = Output.Tuple(app.Project, app.AppId, tool.ToolId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
var toolId = values.Item3;
return $"projects/{project}/locations/us/apps/{appId}/tools/{toolId}";
}),
},
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
Expectation = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationArgs
{
Note = "Expect agent response with chunks",
AgentResponse = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseArgs
{
Role = "agent",
Chunks = new[]
{
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs
{
Text = "Hello again",
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs
{
Text = "Hello again transcript",
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs
{
Blob = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkBlobArgs
{
MimeType = "text/plain",
Data = "c29tZSBibG9iIGRhdGE=",
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs
{
Image = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkImageArgs
{
MimeType = "image/png",
Data = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=",
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs
{
ToolCall = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallArgs
{
Id = "tool-call-id-3",
Tool = Output.Tuple(app.Project, app.AppId, tool.ToolId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
var toolId = values.Item3;
return $"projects/{project}/locations/us/apps/{appId}/tools/{toolId}";
}),
Args =
{
{ "param", "value" },
},
},
},
},
},
},
},
},
},
},
},
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.gcp.ces.App;
import com.pulumi.gcp.ces.AppArgs;
import com.pulumi.gcp.ces.inputs.AppLanguageSettingsArgs;
import com.pulumi.gcp.ces.inputs.AppTimeZoneSettingsArgs;
import com.pulumi.gcp.ces.Tool;
import com.pulumi.gcp.ces.ToolArgs;
import com.pulumi.gcp.ces.inputs.ToolPythonFunctionArgs;
import com.pulumi.gcp.ces.Evaluation;
import com.pulumi.gcp.ces.EvaluationArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnStepArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnStepUserInputArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnStepExpectationArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnStepExpectationToolCallArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnStepExpectationAgentResponseArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkAgentTransferArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnStepExpectationToolCallToolsetToolArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnStepAgentTransferArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnStepExpectationAgentTransferArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnStepExpectationMockToolResponseArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnStepExpectationToolResponseArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnStepExpectationToolResponseToolsetToolArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnStepExpectationUpdatedVariablesArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnStepUserInputToolResponsesArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnStepUserInputToolResponsesToolResponseArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnStepUserInputEventArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnStepUserInputBlobArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnStepUserInputImageArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkBlobArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkImageArgs;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var app = new App("app", AppArgs.builder()
.appId("app-id-full")
.location("us")
.displayName("my-app-full")
.languageSettings(AppLanguageSettingsArgs.builder()
.defaultLanguageCode("en-US")
.build())
.timeZoneSettings(AppTimeZoneSettingsArgs.builder()
.timeZone("America/Los_Angeles")
.build())
.build());
var tool = new Tool("tool", ToolArgs.builder()
.location("us")
.app(app.appId())
.toolId("tool-id-full")
.executionType("SYNCHRONOUS")
.pythonFunction(ToolPythonFunctionArgs.builder()
.name("example_function")
.pythonCode("def example_function() -> int: return 0")
.build())
.build());
var cesEvaluationFull = new Evaluation("cesEvaluationFull", EvaluationArgs.builder()
.evaluationId("evaluation-id-full")
.displayName("my-evaluation-full")
.location("us")
.app(app.appId())
.description("Full evaluation for testing")
.tags(
"test",
"full")
.golden(EvaluationGoldenArgs.builder()
.evaluationExpectations(Output.tuple(app.project(), app.appId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
return String.format("projects/%s/locations/us/apps/%s/evaluationExpectations/dummy-exp", project,appId);
}))
.turns(EvaluationGoldenTurnArgs.builder()
.steps(
EvaluationGoldenTurnStepArgs.builder()
.userInput(EvaluationGoldenTurnStepUserInputArgs.builder()
.text("Hello")
.willContinue(true)
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.userInput(EvaluationGoldenTurnStepUserInputArgs.builder()
.variables(Map.of("key", "value"))
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.expectation(EvaluationGoldenTurnStepExpectationArgs.builder()
.note("Expect tool call")
.toolCall(EvaluationGoldenTurnStepExpectationToolCallArgs.builder()
.id("tool-call-id")
.tool(Output.tuple(app.project(), app.appId(), tool.toolId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
var toolId = values.t3;
return String.format("projects/%s/locations/us/apps/%s/tools/%s", project,appId,toolId);
}))
.args(Map.of("param", "value"))
.build())
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.expectation(EvaluationGoldenTurnStepExpectationArgs.builder()
.note("Expect agent response")
.agentResponse(EvaluationGoldenTurnStepExpectationAgentResponseArgs.builder()
.role("agent")
.chunks(
EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs.builder()
.updatedVariables(Map.of("key", "value"))
.build(),
EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs.builder()
.agentTransfer(EvaluationGoldenTurnStepExpectationAgentResponseChunkAgentTransferArgs.builder()
.targetAgent(Output.tuple(app.project(), app.appId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
return String.format("projects/%s/locations/us/apps/%s/agents/dummy-agent", project,appId);
}))
.build())
.build(),
EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs.builder()
.toolCall(EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallArgs.builder()
.id("tool-call-id-3")
.tool(Output.tuple(app.project(), app.appId(), tool.toolId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
var toolId = values.t3;
return String.format("projects/%s/locations/us/apps/%s/tools/%s", project,appId,toolId);
}))
.build())
.build(),
EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs.builder()
.toolResponse(EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseArgs.builder()
.id("tool-call-id-3")
.response(Map.of("result", "success"))
.tool(Output.tuple(app.project(), app.appId(), tool.toolId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
var toolId = values.t3;
return String.format("projects/%s/locations/us/apps/%s/tools/%s", project,appId,toolId);
}))
.build())
.build())
.build())
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.expectation(EvaluationGoldenTurnStepExpectationArgs.builder()
.note("Expect toolset tool call")
.toolCall(EvaluationGoldenTurnStepExpectationToolCallArgs.builder()
.id("tool-call-id-2")
.toolsetTool(EvaluationGoldenTurnStepExpectationToolCallToolsetToolArgs.builder()
.toolset(Output.tuple(app.project(), app.appId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
return String.format("projects/%s/locations/us/apps/%s/toolsets/dummy-toolset", project,appId);
}))
.toolId("dummy-tool")
.build())
.build())
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.agentTransfer(EvaluationGoldenTurnStepAgentTransferArgs.builder()
.targetAgent(Output.tuple(app.project(), app.appId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
return String.format("projects/%s/locations/us/apps/%s/agents/dummy-agent", project,appId);
}))
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.expectation(EvaluationGoldenTurnStepExpectationArgs.builder()
.agentTransfer(EvaluationGoldenTurnStepExpectationAgentTransferArgs.builder()
.targetAgent(Output.tuple(app.project(), app.appId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
return String.format("projects/%s/locations/us/apps/%s/agents/dummy-agent", project,appId);
}))
.displayName("dummy-agent")
.build())
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.expectation(EvaluationGoldenTurnStepExpectationArgs.builder()
.note("Expect mock tool response")
.mockToolResponse(EvaluationGoldenTurnStepExpectationMockToolResponseArgs.builder()
.id("tool-call-id-4")
.response(Map.of("result", "mocked"))
.tool(Output.tuple(app.project(), app.appId(), tool.toolId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
var toolId = values.t3;
return String.format("projects/%s/locations/us/apps/%s/tools/%s", project,appId,toolId);
}))
.build())
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.expectation(EvaluationGoldenTurnStepExpectationArgs.builder()
.note("Expect tool response")
.toolResponse(EvaluationGoldenTurnStepExpectationToolResponseArgs.builder()
.toolsetTool(EvaluationGoldenTurnStepExpectationToolResponseToolsetToolArgs.builder()
.toolset(Output.tuple(app.project(), app.appId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
return String.format("projects/%s/locations/us/apps/%s/toolsets/dummy-toolset", project,appId);
}))
.toolId("dummy-tool")
.build())
.build())
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.expectation(EvaluationGoldenTurnStepExpectationArgs.builder()
.note("Expect updated variables")
.updatedVariables(EvaluationGoldenTurnStepExpectationUpdatedVariablesArgs.builder()
.notes("Some notes")
.build())
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.userInput(EvaluationGoldenTurnStepUserInputArgs.builder()
.toolResponses(EvaluationGoldenTurnStepUserInputToolResponsesArgs.builder()
.toolResponses(EvaluationGoldenTurnStepUserInputToolResponsesToolResponseArgs.builder()
.id("tool-call-id-5")
.response(Map.of("result", "user-provided"))
.tool(Output.tuple(app.project(), app.appId(), tool.toolId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
var toolId = values.t3;
return String.format("projects/%s/locations/us/apps/%s/tools/%s", project,appId,toolId);
}))
.build())
.build())
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.userInput(EvaluationGoldenTurnStepUserInputArgs.builder()
.event(EvaluationGoldenTurnStepUserInputEventArgs.builder()
.event("my-event")
.build())
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.userInput(EvaluationGoldenTurnStepUserInputArgs.builder()
.dtmf("1234")
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.userInput(EvaluationGoldenTurnStepUserInputArgs.builder()
.audio("c29tZSBhdWRpbyBkYXRh")
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.userInput(EvaluationGoldenTurnStepUserInputArgs.builder()
.blob(EvaluationGoldenTurnStepUserInputBlobArgs.builder()
.mimeType("text/plain")
.data("c29tZSBibG9iIGRhdGE=")
.build())
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.userInput(EvaluationGoldenTurnStepUserInputArgs.builder()
.image(EvaluationGoldenTurnStepUserInputImageArgs.builder()
.mimeType("image/png")
.data("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=")
.build())
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.userInput(EvaluationGoldenTurnStepUserInputArgs.builder()
.toolResponses(EvaluationGoldenTurnStepUserInputToolResponsesArgs.builder()
.toolResponses(EvaluationGoldenTurnStepUserInputToolResponsesToolResponseArgs.builder()
.id("tool-call-id")
.response(Map.of("result", "success"))
.tool(Output.tuple(app.project(), app.appId(), tool.toolId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
var toolId = values.t3;
return String.format("projects/%s/locations/us/apps/%s/tools/%s", project,appId,toolId);
}))
.build())
.build())
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.expectation(EvaluationGoldenTurnStepExpectationArgs.builder()
.note("Expect tool response")
.toolResponse(EvaluationGoldenTurnStepExpectationToolResponseArgs.builder()
.id("tool-call-id")
.response(Map.of("result", "success"))
.tool(Output.tuple(app.project(), app.appId(), tool.toolId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
var toolId = values.t3;
return String.format("projects/%s/locations/us/apps/%s/tools/%s", project,appId,toolId);
}))
.build())
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.expectation(EvaluationGoldenTurnStepExpectationArgs.builder()
.note("Expect agent response with chunks")
.agentResponse(EvaluationGoldenTurnStepExpectationAgentResponseArgs.builder()
.role("agent")
.chunks(
EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs.builder()
.text("Hello again")
.build(),
EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs.builder()
.text("Hello again transcript")
.build(),
EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs.builder()
.blob(EvaluationGoldenTurnStepExpectationAgentResponseChunkBlobArgs.builder()
.mimeType("text/plain")
.data("c29tZSBibG9iIGRhdGE=")
.build())
.build(),
EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs.builder()
.image(EvaluationGoldenTurnStepExpectationAgentResponseChunkImageArgs.builder()
.mimeType("image/png")
.data("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=")
.build())
.build(),
EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs.builder()
.toolCall(EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallArgs.builder()
.id("tool-call-id-3")
.tool(Output.tuple(app.project(), app.appId(), tool.toolId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
var toolId = values.t3;
return String.format("projects/%s/locations/us/apps/%s/tools/%s", project,appId,toolId);
}))
.args(Map.of("param", "value"))
.build())
.build())
.build())
.build())
.build())
.build())
.build())
.build());
}
}
resources:
app:
type: gcp:ces:App
properties:
appId: app-id-full
location: us
displayName: my-app-full
languageSettings:
defaultLanguageCode: en-US
timeZoneSettings:
timeZone: America/Los_Angeles
tool:
type: gcp:ces:Tool
properties:
location: us
app: ${app.appId}
toolId: tool-id-full
executionType: SYNCHRONOUS
pythonFunction:
name: example_function
pythonCode: 'def example_function() -> int: return 0'
cesEvaluationFull:
type: gcp:ces:Evaluation
name: ces_evaluation_full
properties:
evaluationId: evaluation-id-full
displayName: my-evaluation-full
location: us
app: ${app.appId}
description: Full evaluation for testing
tags:
- test
- full
golden:
evaluationExpectations:
- projects/${app.project}/locations/us/apps/${app.appId}/evaluationExpectations/dummy-exp
turns:
- steps:
- userInput:
text: Hello
willContinue: true
- userInput:
variables:
key: value
- expectation:
note: Expect tool call
toolCall:
id: tool-call-id
tool: projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}
args:
param: value
- expectation:
note: Expect agent response
agentResponse:
role: agent
chunks:
- updatedVariables:
key: value
- agentTransfer:
targetAgent: projects/${app.project}/locations/us/apps/${app.appId}/agents/dummy-agent
- toolCall:
id: tool-call-id-3
tool: projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}
- toolResponse:
id: tool-call-id-3
response:
result: success
tool: projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}
- expectation:
note: Expect toolset tool call
toolCall:
id: tool-call-id-2
toolsetTool:
toolset: projects/${app.project}/locations/us/apps/${app.appId}/toolsets/dummy-toolset
toolId: dummy-tool
- agentTransfer:
targetAgent: projects/${app.project}/locations/us/apps/${app.appId}/agents/dummy-agent
- expectation:
agentTransfer:
targetAgent: projects/${app.project}/locations/us/apps/${app.appId}/agents/dummy-agent
displayName: dummy-agent
- expectation:
note: Expect mock tool response
mockToolResponse:
id: tool-call-id-4
response:
result: mocked
tool: projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}
- expectation:
note: Expect tool response
toolResponse:
toolsetTool:
toolset: projects/${app.project}/locations/us/apps/${app.appId}/toolsets/dummy-toolset
toolId: dummy-tool
- expectation:
note: Expect updated variables
updatedVariables:
notes: Some notes
- userInput:
toolResponses:
toolResponses:
- id: tool-call-id-5
response:
result: user-provided
tool: projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}
- userInput:
event:
event: my-event
- userInput:
dtmf: '1234'
- userInput:
audio: c29tZSBhdWRpbyBkYXRh
- userInput:
blob:
mimeType: text/plain
data: c29tZSBibG9iIGRhdGE=
- userInput:
image:
mimeType: image/png
data: iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=
- userInput:
toolResponses:
toolResponses:
- id: tool-call-id
response:
result: success
tool: projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}
- expectation:
note: Expect tool response
toolResponse:
id: tool-call-id
response:
result: success
tool: projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}
- expectation:
note: Expect agent response with chunks
agentResponse:
role: agent
chunks:
- text: Hello again
- text: Hello again transcript
- blob:
mimeType: text/plain
data: c29tZSBibG9iIGRhdGE=
- image:
mimeType: image/png
data: iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=
- toolCall:
id: tool-call-id-3
tool: projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}
args:
param: value
Example coming soon!
Ces Evaluation Toolset
import * as pulumi from "@pulumi/pulumi";
import * as gcp from "@pulumi/gcp";
const app = new gcp.ces.App("app", {
appId: "app-id-toolset",
location: "us",
displayName: "my-app-toolset",
languageSettings: {
defaultLanguageCode: "en-US",
},
timeZoneSettings: {
timeZone: "America/Los_Angeles",
},
});
const cesEvaluationToolset = new gcp.ces.Evaluation("ces_evaluation_toolset", {
evaluationId: "eval-toolset",
displayName: "my-evaluation-toolset",
location: "us",
app: app.appId,
golden: {
turns: [{
steps: [
{
expectation: {
note: "Expect agent response with toolset tool call",
agentResponse: {
role: "agent",
chunks: [{
toolCall: {
id: "tool-call-id",
toolsetTool: {
toolset: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/toolsets/dummy-toolset`,
toolId: "dummy-tool",
},
},
}],
},
},
},
{
expectation: {
note: "Expect agent response with toolset tool response",
agentResponse: {
role: "agent",
chunks: [{
toolResponse: {
id: "tool-call-id",
response: {
result: "success",
},
toolsetTool: {
toolset: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/toolsets/dummy-toolset`,
toolId: "dummy-tool",
},
},
}],
},
},
},
{
expectation: {
note: "Expect mock tool response with toolset tool",
mockToolResponse: {
id: "tool-call-id-2",
response: {
result: "mocked",
},
toolsetTool: {
toolset: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/toolsets/dummy-toolset`,
toolId: "dummy-tool",
},
},
},
},
{
userInput: {
toolResponses: {
toolResponses: [{
id: "tool-call-id-3",
response: {
result: "user-provided",
},
toolsetTool: {
toolset: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/toolsets/dummy-toolset`,
toolId: "dummy-tool",
},
}],
},
},
},
],
}],
},
});
import pulumi
import pulumi_gcp as gcp
app = gcp.ces.App("app",
app_id="app-id-toolset",
location="us",
display_name="my-app-toolset",
language_settings={
"default_language_code": "en-US",
},
time_zone_settings={
"time_zone": "America/Los_Angeles",
})
ces_evaluation_toolset = gcp.ces.Evaluation("ces_evaluation_toolset",
evaluation_id="eval-toolset",
display_name="my-evaluation-toolset",
location="us",
app=app.app_id,
golden={
"turns": [{
"steps": [
{
"expectation": {
"note": "Expect agent response with toolset tool call",
"agent_response": {
"role": "agent",
"chunks": [{
"tool_call": {
"id": "tool-call-id",
"toolset_tool": {
"toolset": pulumi.Output.all(
project=app.project,
app_id=app.app_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/toolsets/dummy-toolset")
,
"tool_id": "dummy-tool",
},
},
}],
},
},
},
{
"expectation": {
"note": "Expect agent response with toolset tool response",
"agent_response": {
"role": "agent",
"chunks": [{
"tool_response": {
"id": "tool-call-id",
"response": {
"result": "success",
},
"toolset_tool": {
"toolset": pulumi.Output.all(
project=app.project,
app_id=app.app_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/toolsets/dummy-toolset")
,
"tool_id": "dummy-tool",
},
},
}],
},
},
},
{
"expectation": {
"note": "Expect mock tool response with toolset tool",
"mock_tool_response": {
"id": "tool-call-id-2",
"response": {
"result": "mocked",
},
"toolset_tool": {
"toolset": pulumi.Output.all(
project=app.project,
app_id=app.app_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/toolsets/dummy-toolset")
,
"tool_id": "dummy-tool",
},
},
},
},
{
"user_input": {
"tool_responses": {
"tool_responses": [{
"id": "tool-call-id-3",
"response": {
"result": "user-provided",
},
"toolset_tool": {
"toolset": pulumi.Output.all(
project=app.project,
app_id=app.app_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/toolsets/dummy-toolset")
,
"tool_id": "dummy-tool",
},
}],
},
},
},
],
}],
})
package main
import (
"fmt"
"github.com/pulumi/pulumi-gcp/sdk/v9/go/gcp/ces"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
app, err := ces.NewApp(ctx, "app", &ces.AppArgs{
AppId: pulumi.String("app-id-toolset"),
Location: pulumi.String("us"),
DisplayName: pulumi.String("my-app-toolset"),
LanguageSettings: &ces.AppLanguageSettingsArgs{
DefaultLanguageCode: pulumi.String("en-US"),
},
TimeZoneSettings: &ces.AppTimeZoneSettingsArgs{
TimeZone: pulumi.String("America/Los_Angeles"),
},
})
if err != nil {
return err
}
_, err = ces.NewEvaluation(ctx, "ces_evaluation_toolset", &ces.EvaluationArgs{
EvaluationId: pulumi.String("eval-toolset"),
DisplayName: pulumi.String("my-evaluation-toolset"),
Location: pulumi.String("us"),
App: app.AppId,
Golden: &ces.EvaluationGoldenArgs{
Turns: ces.EvaluationGoldenTurnArray{
&ces.EvaluationGoldenTurnArgs{
Steps: ces.EvaluationGoldenTurnStepArray{
&ces.EvaluationGoldenTurnStepArgs{
Expectation: &ces.EvaluationGoldenTurnStepExpectationArgs{
Note: pulumi.String("Expect agent response with toolset tool call"),
AgentResponse: &ces.EvaluationGoldenTurnStepExpectationAgentResponseArgs{
Role: pulumi.String("agent"),
Chunks: ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkArray{
&ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs{
ToolCall: &ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallArgs{
Id: pulumi.String("tool-call-id"),
ToolsetTool: &ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallToolsetToolArgs{
Toolset: pulumi.All(app.Project, app.AppId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/toolsets/dummy-toolset", project, appId), nil
}).(pulumi.StringOutput),
ToolId: pulumi.String("dummy-tool"),
},
},
},
},
},
},
},
&ces.EvaluationGoldenTurnStepArgs{
Expectation: &ces.EvaluationGoldenTurnStepExpectationArgs{
Note: pulumi.String("Expect agent response with toolset tool response"),
AgentResponse: &ces.EvaluationGoldenTurnStepExpectationAgentResponseArgs{
Role: pulumi.String("agent"),
Chunks: ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkArray{
&ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs{
ToolResponse: &ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseArgs{
Id: pulumi.String("tool-call-id"),
Response: pulumi.StringMap{
"result": pulumi.String("success"),
},
ToolsetTool: &ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseToolsetToolArgs{
Toolset: pulumi.All(app.Project, app.AppId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/toolsets/dummy-toolset", project, appId), nil
}).(pulumi.StringOutput),
ToolId: pulumi.String("dummy-tool"),
},
},
},
},
},
},
},
&ces.EvaluationGoldenTurnStepArgs{
Expectation: &ces.EvaluationGoldenTurnStepExpectationArgs{
Note: pulumi.String("Expect mock tool response with toolset tool"),
MockToolResponse: &ces.EvaluationGoldenTurnStepExpectationMockToolResponseArgs{
Id: pulumi.String("tool-call-id-2"),
Response: pulumi.StringMap{
"result": pulumi.String("mocked"),
},
ToolsetTool: &ces.EvaluationGoldenTurnStepExpectationMockToolResponseToolsetToolArgs{
Toolset: pulumi.All(app.Project, app.AppId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/toolsets/dummy-toolset", project, appId), nil
}).(pulumi.StringOutput),
ToolId: pulumi.String("dummy-tool"),
},
},
},
},
&ces.EvaluationGoldenTurnStepArgs{
UserInput: &ces.EvaluationGoldenTurnStepUserInputArgs{
ToolResponses: &ces.EvaluationGoldenTurnStepUserInputToolResponsesArgs{
ToolResponses: ces.EvaluationGoldenTurnStepUserInputToolResponsesToolResponseArray{
&ces.EvaluationGoldenTurnStepUserInputToolResponsesToolResponseArgs{
Id: pulumi.String("tool-call-id-3"),
Response: pulumi.StringMap{
"result": pulumi.String("user-provided"),
},
ToolsetTool: &ces.EvaluationGoldenTurnStepUserInputToolResponsesToolResponseToolsetToolArgs{
Toolset: pulumi.All(app.Project, app.AppId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/toolsets/dummy-toolset", project, appId), nil
}).(pulumi.StringOutput),
ToolId: pulumi.String("dummy-tool"),
},
},
},
},
},
},
},
},
},
},
})
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Gcp = Pulumi.Gcp;
return await Deployment.RunAsync(() =>
{
var app = new Gcp.Ces.App("app", new()
{
AppId = "app-id-toolset",
Location = "us",
DisplayName = "my-app-toolset",
LanguageSettings = new Gcp.Ces.Inputs.AppLanguageSettingsArgs
{
DefaultLanguageCode = "en-US",
},
TimeZoneSettings = new Gcp.Ces.Inputs.AppTimeZoneSettingsArgs
{
TimeZone = "America/Los_Angeles",
},
});
var cesEvaluationToolset = new Gcp.Ces.Evaluation("ces_evaluation_toolset", new()
{
EvaluationId = "eval-toolset",
DisplayName = "my-evaluation-toolset",
Location = "us",
App = app.AppId,
Golden = new Gcp.Ces.Inputs.EvaluationGoldenArgs
{
Turns = new[]
{
new Gcp.Ces.Inputs.EvaluationGoldenTurnArgs
{
Steps = new[]
{
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
Expectation = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationArgs
{
Note = "Expect agent response with toolset tool call",
AgentResponse = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseArgs
{
Role = "agent",
Chunks = new[]
{
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs
{
ToolCall = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallArgs
{
Id = "tool-call-id",
ToolsetTool = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallToolsetToolArgs
{
Toolset = Output.Tuple(app.Project, app.AppId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
return $"projects/{project}/locations/us/apps/{appId}/toolsets/dummy-toolset";
}),
ToolId = "dummy-tool",
},
},
},
},
},
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
Expectation = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationArgs
{
Note = "Expect agent response with toolset tool response",
AgentResponse = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseArgs
{
Role = "agent",
Chunks = new[]
{
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs
{
ToolResponse = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseArgs
{
Id = "tool-call-id",
Response =
{
{ "result", "success" },
},
ToolsetTool = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseToolsetToolArgs
{
Toolset = Output.Tuple(app.Project, app.AppId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
return $"projects/{project}/locations/us/apps/{appId}/toolsets/dummy-toolset";
}),
ToolId = "dummy-tool",
},
},
},
},
},
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
Expectation = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationArgs
{
Note = "Expect mock tool response with toolset tool",
MockToolResponse = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationMockToolResponseArgs
{
Id = "tool-call-id-2",
Response =
{
{ "result", "mocked" },
},
ToolsetTool = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationMockToolResponseToolsetToolArgs
{
Toolset = Output.Tuple(app.Project, app.AppId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
return $"projects/{project}/locations/us/apps/{appId}/toolsets/dummy-toolset";
}),
ToolId = "dummy-tool",
},
},
},
},
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
UserInput = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputArgs
{
ToolResponses = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputToolResponsesArgs
{
ToolResponses = new[]
{
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputToolResponsesToolResponseArgs
{
Id = "tool-call-id-3",
Response =
{
{ "result", "user-provided" },
},
ToolsetTool = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputToolResponsesToolResponseToolsetToolArgs
{
Toolset = Output.Tuple(app.Project, app.AppId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
return $"projects/{project}/locations/us/apps/{appId}/toolsets/dummy-toolset";
}),
ToolId = "dummy-tool",
},
},
},
},
},
},
},
},
},
},
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.gcp.ces.App;
import com.pulumi.gcp.ces.AppArgs;
import com.pulumi.gcp.ces.inputs.AppLanguageSettingsArgs;
import com.pulumi.gcp.ces.inputs.AppTimeZoneSettingsArgs;
import com.pulumi.gcp.ces.Evaluation;
import com.pulumi.gcp.ces.EvaluationArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnStepArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnStepExpectationArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnStepExpectationAgentResponseArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallToolsetToolArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseToolsetToolArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnStepExpectationMockToolResponseArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnStepExpectationMockToolResponseToolsetToolArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnStepUserInputArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnStepUserInputToolResponsesArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnStepUserInputToolResponsesToolResponseArgs;
import com.pulumi.gcp.ces.inputs.EvaluationGoldenTurnStepUserInputToolResponsesToolResponseToolsetToolArgs;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var app = new App("app", AppArgs.builder()
.appId("app-id-toolset")
.location("us")
.displayName("my-app-toolset")
.languageSettings(AppLanguageSettingsArgs.builder()
.defaultLanguageCode("en-US")
.build())
.timeZoneSettings(AppTimeZoneSettingsArgs.builder()
.timeZone("America/Los_Angeles")
.build())
.build());
var cesEvaluationToolset = new Evaluation("cesEvaluationToolset", EvaluationArgs.builder()
.evaluationId("eval-toolset")
.displayName("my-evaluation-toolset")
.location("us")
.app(app.appId())
.golden(EvaluationGoldenArgs.builder()
.turns(EvaluationGoldenTurnArgs.builder()
.steps(
EvaluationGoldenTurnStepArgs.builder()
.expectation(EvaluationGoldenTurnStepExpectationArgs.builder()
.note("Expect agent response with toolset tool call")
.agentResponse(EvaluationGoldenTurnStepExpectationAgentResponseArgs.builder()
.role("agent")
.chunks(EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs.builder()
.toolCall(EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallArgs.builder()
.id("tool-call-id")
.toolsetTool(EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallToolsetToolArgs.builder()
.toolset(Output.tuple(app.project(), app.appId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
return String.format("projects/%s/locations/us/apps/%s/toolsets/dummy-toolset", project,appId);
}))
.toolId("dummy-tool")
.build())
.build())
.build())
.build())
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.expectation(EvaluationGoldenTurnStepExpectationArgs.builder()
.note("Expect agent response with toolset tool response")
.agentResponse(EvaluationGoldenTurnStepExpectationAgentResponseArgs.builder()
.role("agent")
.chunks(EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs.builder()
.toolResponse(EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseArgs.builder()
.id("tool-call-id")
.response(Map.of("result", "success"))
.toolsetTool(EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseToolsetToolArgs.builder()
.toolset(Output.tuple(app.project(), app.appId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
return String.format("projects/%s/locations/us/apps/%s/toolsets/dummy-toolset", project,appId);
}))
.toolId("dummy-tool")
.build())
.build())
.build())
.build())
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.expectation(EvaluationGoldenTurnStepExpectationArgs.builder()
.note("Expect mock tool response with toolset tool")
.mockToolResponse(EvaluationGoldenTurnStepExpectationMockToolResponseArgs.builder()
.id("tool-call-id-2")
.response(Map.of("result", "mocked"))
.toolsetTool(EvaluationGoldenTurnStepExpectationMockToolResponseToolsetToolArgs.builder()
.toolset(Output.tuple(app.project(), app.appId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
return String.format("projects/%s/locations/us/apps/%s/toolsets/dummy-toolset", project,appId);
}))
.toolId("dummy-tool")
.build())
.build())
.build())
.build(),
EvaluationGoldenTurnStepArgs.builder()
.userInput(EvaluationGoldenTurnStepUserInputArgs.builder()
.toolResponses(EvaluationGoldenTurnStepUserInputToolResponsesArgs.builder()
.toolResponses(EvaluationGoldenTurnStepUserInputToolResponsesToolResponseArgs.builder()
.id("tool-call-id-3")
.response(Map.of("result", "user-provided"))
.toolsetTool(EvaluationGoldenTurnStepUserInputToolResponsesToolResponseToolsetToolArgs.builder()
.toolset(Output.tuple(app.project(), app.appId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
return String.format("projects/%s/locations/us/apps/%s/toolsets/dummy-toolset", project,appId);
}))
.toolId("dummy-tool")
.build())
.build())
.build())
.build())
.build())
.build())
.build())
.build());
}
}
resources:
app:
type: gcp:ces:App
properties:
appId: app-id-toolset
location: us
displayName: my-app-toolset
languageSettings:
defaultLanguageCode: en-US
timeZoneSettings:
timeZone: America/Los_Angeles
cesEvaluationToolset:
type: gcp:ces:Evaluation
name: ces_evaluation_toolset
properties:
evaluationId: eval-toolset
displayName: my-evaluation-toolset
location: us
app: ${app.appId}
golden:
turns:
- steps:
- expectation:
note: Expect agent response with toolset tool call
agentResponse:
role: agent
chunks:
- toolCall:
id: tool-call-id
toolsetTool:
toolset: projects/${app.project}/locations/us/apps/${app.appId}/toolsets/dummy-toolset
toolId: dummy-tool
- expectation:
note: Expect agent response with toolset tool response
agentResponse:
role: agent
chunks:
- toolResponse:
id: tool-call-id
response:
result: success
toolsetTool:
toolset: projects/${app.project}/locations/us/apps/${app.appId}/toolsets/dummy-toolset
toolId: dummy-tool
- expectation:
note: Expect mock tool response with toolset tool
mockToolResponse:
id: tool-call-id-2
response:
result: mocked
toolsetTool:
toolset: projects/${app.project}/locations/us/apps/${app.appId}/toolsets/dummy-toolset
toolId: dummy-tool
- userInput:
toolResponses:
toolResponses:
- id: tool-call-id-3
response:
result: user-provided
toolsetTool:
toolset: projects/${app.project}/locations/us/apps/${app.appId}/toolsets/dummy-toolset
toolId: dummy-tool
Example coming soon!
Ces Evaluation Scenario Full
import * as pulumi from "@pulumi/pulumi";
import * as gcp from "@pulumi/gcp";
const app = new gcp.ces.App("app", {
appId: "app-id-scenario",
location: "us",
displayName: "my-app-scenario",
languageSettings: {
defaultLanguageCode: "en-US",
},
timeZoneSettings: {
timeZone: "America/Los_Angeles",
},
});
const tool = new gcp.ces.Tool("tool", {
location: "us",
app: app.appId,
toolId: "tool-id-scenario",
executionType: "SYNCHRONOUS",
pythonFunction: {
name: "example_function",
pythonCode: "def example_function() -> int: return 0",
},
});
const cesEvaluationScenarioFull = new gcp.ces.Evaluation("ces_evaluation_scenario_full", {
evaluationId: "eval-scenario-full",
displayName: "my-evaluation-scenario-full",
location: "us",
app: app.appId,
description: "Full evaluation for testing scenario",
tags: [
"test",
"full",
"scenario",
],
scenario: {
task: "Test task",
maxTurns: 5,
rubrics: [pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/rubrics/dummy-rubric`],
userGoalBehavior: "USER_GOAL_SATISFIED",
taskCompletionBehavior: "TASK_SATISFIED",
variableOverrides: {
key: "value",
},
evaluationExpectations: [pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/evaluationExpectations/dummy-exp`],
userFacts: [{
name: "user_name",
value: "John Doe",
}],
scenarioExpectations: [
{
toolExpectation: {
expectedToolCall: {
id: "tool-call-id",
tool: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}`,
args: {
param: "value",
},
},
mockToolResponse: {
id: "tool-call-id",
response: {
result: "mocked",
},
tool: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}`,
},
},
},
{
agentResponse: {
role: "agent",
chunks: [
{
text: "Hello",
},
{
updatedVariables: {
key: "value",
},
},
{
blob: {
mimeType: "text/plain",
data: "c29tZSBibG9iIGRhdGE=",
},
},
{
image: {
mimeType: "image/png",
data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=",
},
},
{
toolCall: {
id: "tool-call-id-3",
tool: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}`,
args: {
param: "value",
},
},
},
{
toolResponse: {
id: "tool-call-id-3",
response: {
result: "success",
},
tool: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}`,
},
},
{
agentTransfer: {
targetAgent: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/agents/dummy-agent`,
},
},
],
},
},
],
},
});
import pulumi
import pulumi_gcp as gcp
app = gcp.ces.App("app",
app_id="app-id-scenario",
location="us",
display_name="my-app-scenario",
language_settings={
"default_language_code": "en-US",
},
time_zone_settings={
"time_zone": "America/Los_Angeles",
})
tool = gcp.ces.Tool("tool",
location="us",
app=app.app_id,
tool_id="tool-id-scenario",
execution_type="SYNCHRONOUS",
python_function={
"name": "example_function",
"python_code": "def example_function() -> int: return 0",
})
ces_evaluation_scenario_full = gcp.ces.Evaluation("ces_evaluation_scenario_full",
evaluation_id="eval-scenario-full",
display_name="my-evaluation-scenario-full",
location="us",
app=app.app_id,
description="Full evaluation for testing scenario",
tags=[
"test",
"full",
"scenario",
],
scenario={
"task": "Test task",
"max_turns": 5,
"rubrics": [pulumi.Output.all(
project=app.project,
app_id=app.app_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/rubrics/dummy-rubric")
],
"user_goal_behavior": "USER_GOAL_SATISFIED",
"task_completion_behavior": "TASK_SATISFIED",
"variable_overrides": {
"key": "value",
},
"evaluation_expectations": [pulumi.Output.all(
project=app.project,
app_id=app.app_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/evaluationExpectations/dummy-exp")
],
"user_facts": [{
"name": "user_name",
"value": "John Doe",
}],
"scenario_expectations": [
{
"tool_expectation": {
"expected_tool_call": {
"id": "tool-call-id",
"tool": pulumi.Output.all(
project=app.project,
app_id=app.app_id,
tool_id=tool.tool_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/tools/{resolved_outputs['tool_id']}")
,
"args": {
"param": "value",
},
},
"mock_tool_response": {
"id": "tool-call-id",
"response": {
"result": "mocked",
},
"tool": pulumi.Output.all(
project=app.project,
app_id=app.app_id,
tool_id=tool.tool_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/tools/{resolved_outputs['tool_id']}")
,
},
},
},
{
"agent_response": {
"role": "agent",
"chunks": [
{
"text": "Hello",
},
{
"updated_variables": {
"key": "value",
},
},
{
"blob": {
"mime_type": "text/plain",
"data": "c29tZSBibG9iIGRhdGE=",
},
},
{
"image": {
"mime_type": "image/png",
"data": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=",
},
},
{
"tool_call": {
"id": "tool-call-id-3",
"tool": pulumi.Output.all(
project=app.project,
app_id=app.app_id,
tool_id=tool.tool_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/tools/{resolved_outputs['tool_id']}")
,
"args": {
"param": "value",
},
},
},
{
"tool_response": {
"id": "tool-call-id-3",
"response": {
"result": "success",
},
"tool": pulumi.Output.all(
project=app.project,
app_id=app.app_id,
tool_id=tool.tool_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/tools/{resolved_outputs['tool_id']}")
,
},
},
{
"agent_transfer": {
"target_agent": pulumi.Output.all(
project=app.project,
app_id=app.app_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/agents/dummy-agent")
,
},
},
],
},
},
],
})
package main
import (
"fmt"
"github.com/pulumi/pulumi-gcp/sdk/v9/go/gcp/ces"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
app, err := ces.NewApp(ctx, "app", &ces.AppArgs{
AppId: pulumi.String("app-id-scenario"),
Location: pulumi.String("us"),
DisplayName: pulumi.String("my-app-scenario"),
LanguageSettings: &ces.AppLanguageSettingsArgs{
DefaultLanguageCode: pulumi.String("en-US"),
},
TimeZoneSettings: &ces.AppTimeZoneSettingsArgs{
TimeZone: pulumi.String("America/Los_Angeles"),
},
})
if err != nil {
return err
}
tool, err := ces.NewTool(ctx, "tool", &ces.ToolArgs{
Location: pulumi.String("us"),
App: app.AppId,
ToolId: pulumi.String("tool-id-scenario"),
ExecutionType: pulumi.String("SYNCHRONOUS"),
PythonFunction: &ces.ToolPythonFunctionArgs{
Name: pulumi.String("example_function"),
PythonCode: pulumi.String("def example_function() -> int: return 0"),
},
})
if err != nil {
return err
}
_, err = ces.NewEvaluation(ctx, "ces_evaluation_scenario_full", &ces.EvaluationArgs{
EvaluationId: pulumi.String("eval-scenario-full"),
DisplayName: pulumi.String("my-evaluation-scenario-full"),
Location: pulumi.String("us"),
App: app.AppId,
Description: pulumi.String("Full evaluation for testing scenario"),
Tags: pulumi.StringArray{
pulumi.String("test"),
pulumi.String("full"),
pulumi.String("scenario"),
},
Scenario: &ces.EvaluationScenarioArgs{
Task: pulumi.String("Test task"),
MaxTurns: pulumi.Int(5),
Rubrics: pulumi.StringArray{
pulumi.All(app.Project, app.AppId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/rubrics/dummy-rubric", project, appId), nil
}).(pulumi.StringOutput),
},
UserGoalBehavior: pulumi.String("USER_GOAL_SATISFIED"),
TaskCompletionBehavior: pulumi.String("TASK_SATISFIED"),
VariableOverrides: pulumi.StringMap{
"key": pulumi.String("value"),
},
EvaluationExpectations: pulumi.StringArray{
pulumi.All(app.Project, app.AppId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/evaluationExpectations/dummy-exp", project, appId), nil
}).(pulumi.StringOutput),
},
UserFacts: ces.EvaluationScenarioUserFactArray{
&ces.EvaluationScenarioUserFactArgs{
Name: pulumi.String("user_name"),
Value: pulumi.String("John Doe"),
},
},
ScenarioExpectations: ces.EvaluationScenarioScenarioExpectationArray{
&ces.EvaluationScenarioScenarioExpectationArgs{
ToolExpectation: &ces.EvaluationScenarioScenarioExpectationToolExpectationArgs{
ExpectedToolCall: &ces.EvaluationScenarioScenarioExpectationToolExpectationExpectedToolCallArgs{
Id: pulumi.String("tool-call-id"),
Tool: pulumi.All(app.Project, app.AppId, tool.ToolId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
toolId := _args[2].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/tools/%v", project, appId, toolId), nil
}).(pulumi.StringOutput),
Args: pulumi.StringMap{
"param": pulumi.String("value"),
},
},
MockToolResponse: &ces.EvaluationScenarioScenarioExpectationToolExpectationMockToolResponseArgs{
Id: pulumi.String("tool-call-id"),
Response: pulumi.StringMap{
"result": pulumi.String("mocked"),
},
Tool: pulumi.All(app.Project, app.AppId, tool.ToolId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
toolId := _args[2].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/tools/%v", project, appId, toolId), nil
}).(pulumi.StringOutput),
},
},
},
&ces.EvaluationScenarioScenarioExpectationArgs{
AgentResponse: &ces.EvaluationScenarioScenarioExpectationAgentResponseArgs{
Role: pulumi.String("agent"),
Chunks: ces.EvaluationScenarioScenarioExpectationAgentResponseChunkArray{
&ces.EvaluationScenarioScenarioExpectationAgentResponseChunkArgs{
Text: pulumi.String("Hello"),
},
&ces.EvaluationScenarioScenarioExpectationAgentResponseChunkArgs{
UpdatedVariables: pulumi.StringMap{
"key": pulumi.String("value"),
},
},
&ces.EvaluationScenarioScenarioExpectationAgentResponseChunkArgs{
Blob: &ces.EvaluationScenarioScenarioExpectationAgentResponseChunkBlobArgs{
MimeType: pulumi.String("text/plain"),
Data: pulumi.String("c29tZSBibG9iIGRhdGE="),
},
},
&ces.EvaluationScenarioScenarioExpectationAgentResponseChunkArgs{
Image: &ces.EvaluationScenarioScenarioExpectationAgentResponseChunkImageArgs{
MimeType: pulumi.String("image/png"),
Data: pulumi.String("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="),
},
},
&ces.EvaluationScenarioScenarioExpectationAgentResponseChunkArgs{
ToolCall: &ces.EvaluationScenarioScenarioExpectationAgentResponseChunkToolCallArgs{
Id: pulumi.String("tool-call-id-3"),
Tool: pulumi.All(app.Project, app.AppId, tool.ToolId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
toolId := _args[2].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/tools/%v", project, appId, toolId), nil
}).(pulumi.StringOutput),
Args: pulumi.StringMap{
"param": pulumi.String("value"),
},
},
},
&ces.EvaluationScenarioScenarioExpectationAgentResponseChunkArgs{
ToolResponse: &ces.EvaluationScenarioScenarioExpectationAgentResponseChunkToolResponseArgs{
Id: pulumi.String("tool-call-id-3"),
Response: pulumi.StringMap{
"result": pulumi.String("success"),
},
Tool: pulumi.All(app.Project, app.AppId, tool.ToolId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
toolId := _args[2].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/tools/%v", project, appId, toolId), nil
}).(pulumi.StringOutput),
},
},
&ces.EvaluationScenarioScenarioExpectationAgentResponseChunkArgs{
AgentTransfer: &ces.EvaluationScenarioScenarioExpectationAgentResponseChunkAgentTransferArgs{
TargetAgent: pulumi.All(app.Project, app.AppId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/agents/dummy-agent", project, appId), nil
}).(pulumi.StringOutput),
},
},
},
},
},
},
},
})
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Gcp = Pulumi.Gcp;
return await Deployment.RunAsync(() =>
{
var app = new Gcp.Ces.App("app", new()
{
AppId = "app-id-scenario",
Location = "us",
DisplayName = "my-app-scenario",
LanguageSettings = new Gcp.Ces.Inputs.AppLanguageSettingsArgs
{
DefaultLanguageCode = "en-US",
},
TimeZoneSettings = new Gcp.Ces.Inputs.AppTimeZoneSettingsArgs
{
TimeZone = "America/Los_Angeles",
},
});
var tool = new Gcp.Ces.Tool("tool", new()
{
Location = "us",
App = app.AppId,
ToolId = "tool-id-scenario",
ExecutionType = "SYNCHRONOUS",
PythonFunction = new Gcp.Ces.Inputs.ToolPythonFunctionArgs
{
Name = "example_function",
PythonCode = "def example_function() -> int: return 0",
},
});
var cesEvaluationScenarioFull = new Gcp.Ces.Evaluation("ces_evaluation_scenario_full", new()
{
EvaluationId = "eval-scenario-full",
DisplayName = "my-evaluation-scenario-full",
Location = "us",
App = app.AppId,
Description = "Full evaluation for testing scenario",
Tags = new[]
{
"test",
"full",
"scenario",
},
Scenario = new Gcp.Ces.Inputs.EvaluationScenarioArgs
{
Task = "Test task",
MaxTurns = 5,
Rubrics = new[]
{
Output.Tuple(app.Project, app.AppId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
return $"projects/{project}/locations/us/apps/{appId}/rubrics/dummy-rubric";
}),
},
UserGoalBehavior = "USER_GOAL_SATISFIED",
TaskCompletionBehavior = "TASK_SATISFIED",
VariableOverrides =
{
{ "key", "value" },
},
EvaluationExpectations = new[]
{
Output.Tuple(app.Project, app.AppId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
return $"projects/{project}/locations/us/apps/{appId}/evaluationExpectations/dummy-exp";
}),
},
UserFacts = new[]
{
new Gcp.Ces.Inputs.EvaluationScenarioUserFactArgs
{
Name = "user_name",
Value = "John Doe",
},
},
ScenarioExpectations = new[]
{
new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationArgs
{
ToolExpectation = new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationToolExpectationArgs
{
ExpectedToolCall = new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationToolExpectationExpectedToolCallArgs
{
Id = "tool-call-id",
Tool = Output.Tuple(app.Project, app.AppId, tool.ToolId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
var toolId = values.Item3;
return $"projects/{project}/locations/us/apps/{appId}/tools/{toolId}";
}),
Args =
{
{ "param", "value" },
},
},
MockToolResponse = new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationToolExpectationMockToolResponseArgs
{
Id = "tool-call-id",
Response =
{
{ "result", "mocked" },
},
Tool = Output.Tuple(app.Project, app.AppId, tool.ToolId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
var toolId = values.Item3;
return $"projects/{project}/locations/us/apps/{appId}/tools/{toolId}";
}),
},
},
},
new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationArgs
{
AgentResponse = new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationAgentResponseArgs
{
Role = "agent",
Chunks = new[]
{
new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkArgs
{
Text = "Hello",
},
new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkArgs
{
UpdatedVariables =
{
{ "key", "value" },
},
},
new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkArgs
{
Blob = new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkBlobArgs
{
MimeType = "text/plain",
Data = "c29tZSBibG9iIGRhdGE=",
},
},
new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkArgs
{
Image = new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkImageArgs
{
MimeType = "image/png",
Data = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=",
},
},
new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkArgs
{
ToolCall = new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkToolCallArgs
{
Id = "tool-call-id-3",
Tool = Output.Tuple(app.Project, app.AppId, tool.ToolId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
var toolId = values.Item3;
return $"projects/{project}/locations/us/apps/{appId}/tools/{toolId}";
}),
Args =
{
{ "param", "value" },
},
},
},
new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkArgs
{
ToolResponse = new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkToolResponseArgs
{
Id = "tool-call-id-3",
Response =
{
{ "result", "success" },
},
Tool = Output.Tuple(app.Project, app.AppId, tool.ToolId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
var toolId = values.Item3;
return $"projects/{project}/locations/us/apps/{appId}/tools/{toolId}";
}),
},
},
new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkArgs
{
AgentTransfer = new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkAgentTransferArgs
{
TargetAgent = Output.Tuple(app.Project, app.AppId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
return $"projects/{project}/locations/us/apps/{appId}/agents/dummy-agent";
}),
},
},
},
},
},
},
},
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.gcp.ces.App;
import com.pulumi.gcp.ces.AppArgs;
import com.pulumi.gcp.ces.inputs.AppLanguageSettingsArgs;
import com.pulumi.gcp.ces.inputs.AppTimeZoneSettingsArgs;
import com.pulumi.gcp.ces.Tool;
import com.pulumi.gcp.ces.ToolArgs;
import com.pulumi.gcp.ces.inputs.ToolPythonFunctionArgs;
import com.pulumi.gcp.ces.Evaluation;
import com.pulumi.gcp.ces.EvaluationArgs;
import com.pulumi.gcp.ces.inputs.EvaluationScenarioArgs;
import com.pulumi.gcp.ces.inputs.EvaluationScenarioUserFactArgs;
import com.pulumi.gcp.ces.inputs.EvaluationScenarioScenarioExpectationArgs;
import com.pulumi.gcp.ces.inputs.EvaluationScenarioScenarioExpectationToolExpectationArgs;
import com.pulumi.gcp.ces.inputs.EvaluationScenarioScenarioExpectationToolExpectationExpectedToolCallArgs;
import com.pulumi.gcp.ces.inputs.EvaluationScenarioScenarioExpectationToolExpectationMockToolResponseArgs;
import com.pulumi.gcp.ces.inputs.EvaluationScenarioScenarioExpectationAgentResponseArgs;
import com.pulumi.gcp.ces.inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkArgs;
import com.pulumi.gcp.ces.inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkBlobArgs;
import com.pulumi.gcp.ces.inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkImageArgs;
import com.pulumi.gcp.ces.inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkToolCallArgs;
import com.pulumi.gcp.ces.inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkToolResponseArgs;
import com.pulumi.gcp.ces.inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkAgentTransferArgs;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var app = new App("app", AppArgs.builder()
.appId("app-id-scenario")
.location("us")
.displayName("my-app-scenario")
.languageSettings(AppLanguageSettingsArgs.builder()
.defaultLanguageCode("en-US")
.build())
.timeZoneSettings(AppTimeZoneSettingsArgs.builder()
.timeZone("America/Los_Angeles")
.build())
.build());
var tool = new Tool("tool", ToolArgs.builder()
.location("us")
.app(app.appId())
.toolId("tool-id-scenario")
.executionType("SYNCHRONOUS")
.pythonFunction(ToolPythonFunctionArgs.builder()
.name("example_function")
.pythonCode("def example_function() -> int: return 0")
.build())
.build());
var cesEvaluationScenarioFull = new Evaluation("cesEvaluationScenarioFull", EvaluationArgs.builder()
.evaluationId("eval-scenario-full")
.displayName("my-evaluation-scenario-full")
.location("us")
.app(app.appId())
.description("Full evaluation for testing scenario")
.tags(
"test",
"full",
"scenario")
.scenario(EvaluationScenarioArgs.builder()
.task("Test task")
.maxTurns(5)
.rubrics(Output.tuple(app.project(), app.appId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
return String.format("projects/%s/locations/us/apps/%s/rubrics/dummy-rubric", project,appId);
}))
.userGoalBehavior("USER_GOAL_SATISFIED")
.taskCompletionBehavior("TASK_SATISFIED")
.variableOverrides(Map.of("key", "value"))
.evaluationExpectations(Output.tuple(app.project(), app.appId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
return String.format("projects/%s/locations/us/apps/%s/evaluationExpectations/dummy-exp", project,appId);
}))
.userFacts(EvaluationScenarioUserFactArgs.builder()
.name("user_name")
.value("John Doe")
.build())
.scenarioExpectations(
EvaluationScenarioScenarioExpectationArgs.builder()
.toolExpectation(EvaluationScenarioScenarioExpectationToolExpectationArgs.builder()
.expectedToolCall(EvaluationScenarioScenarioExpectationToolExpectationExpectedToolCallArgs.builder()
.id("tool-call-id")
.tool(Output.tuple(app.project(), app.appId(), tool.toolId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
var toolId = values.t3;
return String.format("projects/%s/locations/us/apps/%s/tools/%s", project,appId,toolId);
}))
.args(Map.of("param", "value"))
.build())
.mockToolResponse(EvaluationScenarioScenarioExpectationToolExpectationMockToolResponseArgs.builder()
.id("tool-call-id")
.response(Map.of("result", "mocked"))
.tool(Output.tuple(app.project(), app.appId(), tool.toolId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
var toolId = values.t3;
return String.format("projects/%s/locations/us/apps/%s/tools/%s", project,appId,toolId);
}))
.build())
.build())
.build(),
EvaluationScenarioScenarioExpectationArgs.builder()
.agentResponse(EvaluationScenarioScenarioExpectationAgentResponseArgs.builder()
.role("agent")
.chunks(
EvaluationScenarioScenarioExpectationAgentResponseChunkArgs.builder()
.text("Hello")
.build(),
EvaluationScenarioScenarioExpectationAgentResponseChunkArgs.builder()
.updatedVariables(Map.of("key", "value"))
.build(),
EvaluationScenarioScenarioExpectationAgentResponseChunkArgs.builder()
.blob(EvaluationScenarioScenarioExpectationAgentResponseChunkBlobArgs.builder()
.mimeType("text/plain")
.data("c29tZSBibG9iIGRhdGE=")
.build())
.build(),
EvaluationScenarioScenarioExpectationAgentResponseChunkArgs.builder()
.image(EvaluationScenarioScenarioExpectationAgentResponseChunkImageArgs.builder()
.mimeType("image/png")
.data("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=")
.build())
.build(),
EvaluationScenarioScenarioExpectationAgentResponseChunkArgs.builder()
.toolCall(EvaluationScenarioScenarioExpectationAgentResponseChunkToolCallArgs.builder()
.id("tool-call-id-3")
.tool(Output.tuple(app.project(), app.appId(), tool.toolId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
var toolId = values.t3;
return String.format("projects/%s/locations/us/apps/%s/tools/%s", project,appId,toolId);
}))
.args(Map.of("param", "value"))
.build())
.build(),
EvaluationScenarioScenarioExpectationAgentResponseChunkArgs.builder()
.toolResponse(EvaluationScenarioScenarioExpectationAgentResponseChunkToolResponseArgs.builder()
.id("tool-call-id-3")
.response(Map.of("result", "success"))
.tool(Output.tuple(app.project(), app.appId(), tool.toolId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
var toolId = values.t3;
return String.format("projects/%s/locations/us/apps/%s/tools/%s", project,appId,toolId);
}))
.build())
.build(),
EvaluationScenarioScenarioExpectationAgentResponseChunkArgs.builder()
.agentTransfer(EvaluationScenarioScenarioExpectationAgentResponseChunkAgentTransferArgs.builder()
.targetAgent(Output.tuple(app.project(), app.appId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
return String.format("projects/%s/locations/us/apps/%s/agents/dummy-agent", project,appId);
}))
.build())
.build())
.build())
.build())
.build())
.build());
}
}
resources:
app:
type: gcp:ces:App
properties:
appId: app-id-scenario
location: us
displayName: my-app-scenario
languageSettings:
defaultLanguageCode: en-US
timeZoneSettings:
timeZone: America/Los_Angeles
tool:
type: gcp:ces:Tool
properties:
location: us
app: ${app.appId}
toolId: tool-id-scenario
executionType: SYNCHRONOUS
pythonFunction:
name: example_function
pythonCode: 'def example_function() -> int: return 0'
cesEvaluationScenarioFull:
type: gcp:ces:Evaluation
name: ces_evaluation_scenario_full
properties:
evaluationId: eval-scenario-full
displayName: my-evaluation-scenario-full
location: us
app: ${app.appId}
description: Full evaluation for testing scenario
tags:
- test
- full
- scenario
scenario:
task: Test task
maxTurns: 5
rubrics:
- projects/${app.project}/locations/us/apps/${app.appId}/rubrics/dummy-rubric
userGoalBehavior: USER_GOAL_SATISFIED
taskCompletionBehavior: TASK_SATISFIED
variableOverrides:
key: value
evaluationExpectations:
- projects/${app.project}/locations/us/apps/${app.appId}/evaluationExpectations/dummy-exp
userFacts:
- name: user_name
value: John Doe
scenarioExpectations:
- toolExpectation:
expectedToolCall:
id: tool-call-id
tool: projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}
args:
param: value
mockToolResponse:
id: tool-call-id
response:
result: mocked
tool: projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}
- agentResponse:
role: agent
chunks:
- text: Hello
- updatedVariables:
key: value
- blob:
mimeType: text/plain
data: c29tZSBibG9iIGRhdGE=
- image:
mimeType: image/png
data: iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=
- toolCall:
id: tool-call-id-3
tool: projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}
args:
param: value
- toolResponse:
id: tool-call-id-3
response:
result: success
tool: projects/${app.project}/locations/us/apps/${app.appId}/tools/${tool.toolId}
- agentTransfer:
targetAgent: projects/${app.project}/locations/us/apps/${app.appId}/agents/dummy-agent
Example coming soon!
Ces Evaluation Scenario Toolset
import * as pulumi from "@pulumi/pulumi";
import * as gcp from "@pulumi/gcp";
const app = new gcp.ces.App("app", {
appId: "app-id-scenario-ts",
location: "us",
displayName: "my-app-scenario-ts",
languageSettings: {
defaultLanguageCode: "en-US",
},
timeZoneSettings: {
timeZone: "America/Los_Angeles",
},
});
const toolset = new gcp.ces.Toolset("toolset", {
toolsetId: "ts-scen",
location: "us",
app: app.appId,
displayName: "Basic toolset display name",
description: "Test description",
executionType: "SYNCHRONOUS",
openApiToolset: {
openApiSchema: `openapi: 3.0.0
info:
title: My Sample API
version: 1.0.0
description: A simple API example
servers:
- url: https://api.example.com/v1
paths: {}
`,
ignoreUnknownFields: false,
},
});
const cesEvaluationScenarioToolset = new gcp.ces.Evaluation("ces_evaluation_scenario_toolset", {
evaluationId: "eval-scen-ts",
displayName: "my-evaluation-scenario-toolset",
location: "us",
app: app.appId,
description: "Full evaluation for testing scenario with toolset",
tags: [
"test",
"full",
"scenario",
"toolset",
],
scenario: {
task: "Test task",
maxTurns: 5,
rubrics: [pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/rubrics/dummy-rubric`],
userGoalBehavior: "USER_GOAL_SATISFIED",
taskCompletionBehavior: "TASK_SATISFIED",
variableOverrides: {
key: "value",
},
evaluationExpectations: [pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/evaluationExpectations/dummy-exp`],
userFacts: [{
name: "user_name",
value: "John Doe",
}],
scenarioExpectations: [
{
toolExpectation: {
expectedToolCall: {
id: "tool-call-id",
toolsetTool: {
toolset: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/toolsets/${toolset.toolsetId}`,
toolId: "dummy-tool",
},
args: {
param: "value",
},
},
mockToolResponse: {
id: "tool-call-id",
response: {
result: "mocked",
},
toolsetTool: {
toolset: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/toolsets/${toolset.toolsetId}`,
toolId: "dummy-tool",
},
},
},
},
{
agentResponse: {
role: "agent",
chunks: [
{
text: "Hello",
},
{
toolCall: {
id: "tool-call-id-3",
toolsetTool: {
toolset: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/toolsets/${toolset.toolsetId}`,
toolId: "dummy-tool",
},
args: {
param: "value",
},
},
},
{
toolResponse: {
id: "tool-call-id-3",
response: {
result: "success",
},
toolsetTool: {
toolset: pulumi.interpolate`projects/${app.project}/locations/us/apps/${app.appId}/toolsets/${toolset.toolsetId}`,
toolId: "dummy-tool",
},
},
},
],
},
},
],
},
});
import pulumi
import pulumi_gcp as gcp
app = gcp.ces.App("app",
app_id="app-id-scenario-ts",
location="us",
display_name="my-app-scenario-ts",
language_settings={
"default_language_code": "en-US",
},
time_zone_settings={
"time_zone": "America/Los_Angeles",
})
toolset = gcp.ces.Toolset("toolset",
toolset_id="ts-scen",
location="us",
app=app.app_id,
display_name="Basic toolset display name",
description="Test description",
execution_type="SYNCHRONOUS",
open_api_toolset={
"open_api_schema": """openapi: 3.0.0
info:
title: My Sample API
version: 1.0.0
description: A simple API example
servers:
- url: https://api.example.com/v1
paths: {}
""",
"ignore_unknown_fields": False,
})
ces_evaluation_scenario_toolset = gcp.ces.Evaluation("ces_evaluation_scenario_toolset",
evaluation_id="eval-scen-ts",
display_name="my-evaluation-scenario-toolset",
location="us",
app=app.app_id,
description="Full evaluation for testing scenario with toolset",
tags=[
"test",
"full",
"scenario",
"toolset",
],
scenario={
"task": "Test task",
"max_turns": 5,
"rubrics": [pulumi.Output.all(
project=app.project,
app_id=app.app_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/rubrics/dummy-rubric")
],
"user_goal_behavior": "USER_GOAL_SATISFIED",
"task_completion_behavior": "TASK_SATISFIED",
"variable_overrides": {
"key": "value",
},
"evaluation_expectations": [pulumi.Output.all(
project=app.project,
app_id=app.app_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/evaluationExpectations/dummy-exp")
],
"user_facts": [{
"name": "user_name",
"value": "John Doe",
}],
"scenario_expectations": [
{
"tool_expectation": {
"expected_tool_call": {
"id": "tool-call-id",
"toolset_tool": {
"toolset": pulumi.Output.all(
project=app.project,
app_id=app.app_id,
toolset_id=toolset.toolset_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/toolsets/{resolved_outputs['toolset_id']}")
,
"tool_id": "dummy-tool",
},
"args": {
"param": "value",
},
},
"mock_tool_response": {
"id": "tool-call-id",
"response": {
"result": "mocked",
},
"toolset_tool": {
"toolset": pulumi.Output.all(
project=app.project,
app_id=app.app_id,
toolset_id=toolset.toolset_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/toolsets/{resolved_outputs['toolset_id']}")
,
"tool_id": "dummy-tool",
},
},
},
},
{
"agent_response": {
"role": "agent",
"chunks": [
{
"text": "Hello",
},
{
"tool_call": {
"id": "tool-call-id-3",
"toolset_tool": {
"toolset": pulumi.Output.all(
project=app.project,
app_id=app.app_id,
toolset_id=toolset.toolset_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/toolsets/{resolved_outputs['toolset_id']}")
,
"tool_id": "dummy-tool",
},
"args": {
"param": "value",
},
},
},
{
"tool_response": {
"id": "tool-call-id-3",
"response": {
"result": "success",
},
"toolset_tool": {
"toolset": pulumi.Output.all(
project=app.project,
app_id=app.app_id,
toolset_id=toolset.toolset_id
).apply(lambda resolved_outputs: f"projects/{resolved_outputs['project']}/locations/us/apps/{resolved_outputs['app_id']}/toolsets/{resolved_outputs['toolset_id']}")
,
"tool_id": "dummy-tool",
},
},
},
],
},
},
],
})
package main
import (
"fmt"
"github.com/pulumi/pulumi-gcp/sdk/v9/go/gcp/ces"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
app, err := ces.NewApp(ctx, "app", &ces.AppArgs{
AppId: pulumi.String("app-id-scenario-ts"),
Location: pulumi.String("us"),
DisplayName: pulumi.String("my-app-scenario-ts"),
LanguageSettings: &ces.AppLanguageSettingsArgs{
DefaultLanguageCode: pulumi.String("en-US"),
},
TimeZoneSettings: &ces.AppTimeZoneSettingsArgs{
TimeZone: pulumi.String("America/Los_Angeles"),
},
})
if err != nil {
return err
}
toolset, err := ces.NewToolset(ctx, "toolset", &ces.ToolsetArgs{
ToolsetId: pulumi.String("ts-scen"),
Location: pulumi.String("us"),
App: app.AppId,
DisplayName: pulumi.String("Basic toolset display name"),
Description: pulumi.String("Test description"),
ExecutionType: pulumi.String("SYNCHRONOUS"),
OpenApiToolset: &ces.ToolsetOpenApiToolsetArgs{
OpenApiSchema: pulumi.String(`openapi: 3.0.0
info:
title: My Sample API
version: 1.0.0
description: A simple API example
servers:
- url: https://api.example.com/v1
paths: {}
`),
IgnoreUnknownFields: pulumi.Bool(false),
},
})
if err != nil {
return err
}
_, err = ces.NewEvaluation(ctx, "ces_evaluation_scenario_toolset", &ces.EvaluationArgs{
EvaluationId: pulumi.String("eval-scen-ts"),
DisplayName: pulumi.String("my-evaluation-scenario-toolset"),
Location: pulumi.String("us"),
App: app.AppId,
Description: pulumi.String("Full evaluation for testing scenario with toolset"),
Tags: pulumi.StringArray{
pulumi.String("test"),
pulumi.String("full"),
pulumi.String("scenario"),
pulumi.String("toolset"),
},
Scenario: &ces.EvaluationScenarioArgs{
Task: pulumi.String("Test task"),
MaxTurns: pulumi.Int(5),
Rubrics: pulumi.StringArray{
pulumi.All(app.Project, app.AppId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/rubrics/dummy-rubric", project, appId), nil
}).(pulumi.StringOutput),
},
UserGoalBehavior: pulumi.String("USER_GOAL_SATISFIED"),
TaskCompletionBehavior: pulumi.String("TASK_SATISFIED"),
VariableOverrides: pulumi.StringMap{
"key": pulumi.String("value"),
},
EvaluationExpectations: pulumi.StringArray{
pulumi.All(app.Project, app.AppId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/evaluationExpectations/dummy-exp", project, appId), nil
}).(pulumi.StringOutput),
},
UserFacts: ces.EvaluationScenarioUserFactArray{
&ces.EvaluationScenarioUserFactArgs{
Name: pulumi.String("user_name"),
Value: pulumi.String("John Doe"),
},
},
ScenarioExpectations: ces.EvaluationScenarioScenarioExpectationArray{
&ces.EvaluationScenarioScenarioExpectationArgs{
ToolExpectation: &ces.EvaluationScenarioScenarioExpectationToolExpectationArgs{
ExpectedToolCall: &ces.EvaluationScenarioScenarioExpectationToolExpectationExpectedToolCallArgs{
Id: pulumi.String("tool-call-id"),
ToolsetTool: &ces.EvaluationScenarioScenarioExpectationToolExpectationExpectedToolCallToolsetToolArgs{
Toolset: pulumi.All(app.Project, app.AppId, toolset.ToolsetId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
toolsetId := _args[2].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/toolsets/%v", project, appId, toolsetId), nil
}).(pulumi.StringOutput),
ToolId: pulumi.String("dummy-tool"),
},
Args: pulumi.StringMap{
"param": pulumi.String("value"),
},
},
MockToolResponse: &ces.EvaluationScenarioScenarioExpectationToolExpectationMockToolResponseArgs{
Id: pulumi.String("tool-call-id"),
Response: pulumi.StringMap{
"result": pulumi.String("mocked"),
},
ToolsetTool: &ces.EvaluationScenarioScenarioExpectationToolExpectationMockToolResponseToolsetToolArgs{
Toolset: pulumi.All(app.Project, app.AppId, toolset.ToolsetId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
toolsetId := _args[2].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/toolsets/%v", project, appId, toolsetId), nil
}).(pulumi.StringOutput),
ToolId: pulumi.String("dummy-tool"),
},
},
},
},
&ces.EvaluationScenarioScenarioExpectationArgs{
AgentResponse: &ces.EvaluationScenarioScenarioExpectationAgentResponseArgs{
Role: pulumi.String("agent"),
Chunks: ces.EvaluationScenarioScenarioExpectationAgentResponseChunkArray{
&ces.EvaluationScenarioScenarioExpectationAgentResponseChunkArgs{
Text: pulumi.String("Hello"),
},
&ces.EvaluationScenarioScenarioExpectationAgentResponseChunkArgs{
ToolCall: &ces.EvaluationScenarioScenarioExpectationAgentResponseChunkToolCallArgs{
Id: pulumi.String("tool-call-id-3"),
ToolsetTool: &ces.EvaluationScenarioScenarioExpectationAgentResponseChunkToolCallToolsetToolArgs{
Toolset: pulumi.All(app.Project, app.AppId, toolset.ToolsetId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
toolsetId := _args[2].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/toolsets/%v", project, appId, toolsetId), nil
}).(pulumi.StringOutput),
ToolId: pulumi.String("dummy-tool"),
},
Args: pulumi.StringMap{
"param": pulumi.String("value"),
},
},
},
&ces.EvaluationScenarioScenarioExpectationAgentResponseChunkArgs{
ToolResponse: &ces.EvaluationScenarioScenarioExpectationAgentResponseChunkToolResponseArgs{
Id: pulumi.String("tool-call-id-3"),
Response: pulumi.StringMap{
"result": pulumi.String("success"),
},
ToolsetTool: &ces.EvaluationScenarioScenarioExpectationAgentResponseChunkToolResponseToolsetToolArgs{
Toolset: pulumi.All(app.Project, app.AppId, toolset.ToolsetId).ApplyT(func(_args []interface{}) (string, error) {
project := _args[0].(string)
appId := _args[1].(string)
toolsetId := _args[2].(string)
return fmt.Sprintf("projects/%v/locations/us/apps/%v/toolsets/%v", project, appId, toolsetId), nil
}).(pulumi.StringOutput),
ToolId: pulumi.String("dummy-tool"),
},
},
},
},
},
},
},
},
})
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Gcp = Pulumi.Gcp;
return await Deployment.RunAsync(() =>
{
var app = new Gcp.Ces.App("app", new()
{
AppId = "app-id-scenario-ts",
Location = "us",
DisplayName = "my-app-scenario-ts",
LanguageSettings = new Gcp.Ces.Inputs.AppLanguageSettingsArgs
{
DefaultLanguageCode = "en-US",
},
TimeZoneSettings = new Gcp.Ces.Inputs.AppTimeZoneSettingsArgs
{
TimeZone = "America/Los_Angeles",
},
});
var toolset = new Gcp.Ces.Toolset("toolset", new()
{
ToolsetId = "ts-scen",
Location = "us",
App = app.AppId,
DisplayName = "Basic toolset display name",
Description = "Test description",
ExecutionType = "SYNCHRONOUS",
OpenApiToolset = new Gcp.Ces.Inputs.ToolsetOpenApiToolsetArgs
{
OpenApiSchema = @"openapi: 3.0.0
info:
title: My Sample API
version: 1.0.0
description: A simple API example
servers:
- url: https://api.example.com/v1
paths: {}
",
IgnoreUnknownFields = false,
},
});
var cesEvaluationScenarioToolset = new Gcp.Ces.Evaluation("ces_evaluation_scenario_toolset", new()
{
EvaluationId = "eval-scen-ts",
DisplayName = "my-evaluation-scenario-toolset",
Location = "us",
App = app.AppId,
Description = "Full evaluation for testing scenario with toolset",
Tags = new[]
{
"test",
"full",
"scenario",
"toolset",
},
Scenario = new Gcp.Ces.Inputs.EvaluationScenarioArgs
{
Task = "Test task",
MaxTurns = 5,
Rubrics = new[]
{
Output.Tuple(app.Project, app.AppId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
return $"projects/{project}/locations/us/apps/{appId}/rubrics/dummy-rubric";
}),
},
UserGoalBehavior = "USER_GOAL_SATISFIED",
TaskCompletionBehavior = "TASK_SATISFIED",
VariableOverrides =
{
{ "key", "value" },
},
EvaluationExpectations = new[]
{
Output.Tuple(app.Project, app.AppId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
return $"projects/{project}/locations/us/apps/{appId}/evaluationExpectations/dummy-exp";
}),
},
UserFacts = new[]
{
new Gcp.Ces.Inputs.EvaluationScenarioUserFactArgs
{
Name = "user_name",
Value = "John Doe",
},
},
ScenarioExpectations = new[]
{
new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationArgs
{
ToolExpectation = new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationToolExpectationArgs
{
ExpectedToolCall = new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationToolExpectationExpectedToolCallArgs
{
Id = "tool-call-id",
ToolsetTool = new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationToolExpectationExpectedToolCallToolsetToolArgs
{
Toolset = Output.Tuple(app.Project, app.AppId, toolset.ToolsetId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
var toolsetId = values.Item3;
return $"projects/{project}/locations/us/apps/{appId}/toolsets/{toolsetId}";
}),
ToolId = "dummy-tool",
},
Args =
{
{ "param", "value" },
},
},
MockToolResponse = new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationToolExpectationMockToolResponseArgs
{
Id = "tool-call-id",
Response =
{
{ "result", "mocked" },
},
ToolsetTool = new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationToolExpectationMockToolResponseToolsetToolArgs
{
Toolset = Output.Tuple(app.Project, app.AppId, toolset.ToolsetId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
var toolsetId = values.Item3;
return $"projects/{project}/locations/us/apps/{appId}/toolsets/{toolsetId}";
}),
ToolId = "dummy-tool",
},
},
},
},
new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationArgs
{
AgentResponse = new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationAgentResponseArgs
{
Role = "agent",
Chunks = new[]
{
new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkArgs
{
Text = "Hello",
},
new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkArgs
{
ToolCall = new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkToolCallArgs
{
Id = "tool-call-id-3",
ToolsetTool = new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkToolCallToolsetToolArgs
{
Toolset = Output.Tuple(app.Project, app.AppId, toolset.ToolsetId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
var toolsetId = values.Item3;
return $"projects/{project}/locations/us/apps/{appId}/toolsets/{toolsetId}";
}),
ToolId = "dummy-tool",
},
Args =
{
{ "param", "value" },
},
},
},
new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkArgs
{
ToolResponse = new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkToolResponseArgs
{
Id = "tool-call-id-3",
Response =
{
{ "result", "success" },
},
ToolsetTool = new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkToolResponseToolsetToolArgs
{
Toolset = Output.Tuple(app.Project, app.AppId, toolset.ToolsetId).Apply(values =>
{
var project = values.Item1;
var appId = values.Item2;
var toolsetId = values.Item3;
return $"projects/{project}/locations/us/apps/{appId}/toolsets/{toolsetId}";
}),
ToolId = "dummy-tool",
},
},
},
},
},
},
},
},
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.gcp.ces.App;
import com.pulumi.gcp.ces.AppArgs;
import com.pulumi.gcp.ces.inputs.AppLanguageSettingsArgs;
import com.pulumi.gcp.ces.inputs.AppTimeZoneSettingsArgs;
import com.pulumi.gcp.ces.Toolset;
import com.pulumi.gcp.ces.ToolsetArgs;
import com.pulumi.gcp.ces.inputs.ToolsetOpenApiToolsetArgs;
import com.pulumi.gcp.ces.Evaluation;
import com.pulumi.gcp.ces.EvaluationArgs;
import com.pulumi.gcp.ces.inputs.EvaluationScenarioArgs;
import com.pulumi.gcp.ces.inputs.EvaluationScenarioUserFactArgs;
import com.pulumi.gcp.ces.inputs.EvaluationScenarioScenarioExpectationArgs;
import com.pulumi.gcp.ces.inputs.EvaluationScenarioScenarioExpectationToolExpectationArgs;
import com.pulumi.gcp.ces.inputs.EvaluationScenarioScenarioExpectationToolExpectationExpectedToolCallArgs;
import com.pulumi.gcp.ces.inputs.EvaluationScenarioScenarioExpectationToolExpectationExpectedToolCallToolsetToolArgs;
import com.pulumi.gcp.ces.inputs.EvaluationScenarioScenarioExpectationToolExpectationMockToolResponseArgs;
import com.pulumi.gcp.ces.inputs.EvaluationScenarioScenarioExpectationToolExpectationMockToolResponseToolsetToolArgs;
import com.pulumi.gcp.ces.inputs.EvaluationScenarioScenarioExpectationAgentResponseArgs;
import com.pulumi.gcp.ces.inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkArgs;
import com.pulumi.gcp.ces.inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkToolCallArgs;
import com.pulumi.gcp.ces.inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkToolCallToolsetToolArgs;
import com.pulumi.gcp.ces.inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkToolResponseArgs;
import com.pulumi.gcp.ces.inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkToolResponseToolsetToolArgs;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
var app = new App("app", AppArgs.builder()
.appId("app-id-scenario-ts")
.location("us")
.displayName("my-app-scenario-ts")
.languageSettings(AppLanguageSettingsArgs.builder()
.defaultLanguageCode("en-US")
.build())
.timeZoneSettings(AppTimeZoneSettingsArgs.builder()
.timeZone("America/Los_Angeles")
.build())
.build());
var toolset = new Toolset("toolset", ToolsetArgs.builder()
.toolsetId("ts-scen")
.location("us")
.app(app.appId())
.displayName("Basic toolset display name")
.description("Test description")
.executionType("SYNCHRONOUS")
.openApiToolset(ToolsetOpenApiToolsetArgs.builder()
.openApiSchema("""
openapi: 3.0.0
info:
title: My Sample API
version: 1.0.0
description: A simple API example
servers:
- url: https://api.example.com/v1
paths: {}
""")
.ignoreUnknownFields(false)
.build())
.build());
var cesEvaluationScenarioToolset = new Evaluation("cesEvaluationScenarioToolset", EvaluationArgs.builder()
.evaluationId("eval-scen-ts")
.displayName("my-evaluation-scenario-toolset")
.location("us")
.app(app.appId())
.description("Full evaluation for testing scenario with toolset")
.tags(
"test",
"full",
"scenario",
"toolset")
.scenario(EvaluationScenarioArgs.builder()
.task("Test task")
.maxTurns(5)
.rubrics(Output.tuple(app.project(), app.appId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
return String.format("projects/%s/locations/us/apps/%s/rubrics/dummy-rubric", project,appId);
}))
.userGoalBehavior("USER_GOAL_SATISFIED")
.taskCompletionBehavior("TASK_SATISFIED")
.variableOverrides(Map.of("key", "value"))
.evaluationExpectations(Output.tuple(app.project(), app.appId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
return String.format("projects/%s/locations/us/apps/%s/evaluationExpectations/dummy-exp", project,appId);
}))
.userFacts(EvaluationScenarioUserFactArgs.builder()
.name("user_name")
.value("John Doe")
.build())
.scenarioExpectations(
EvaluationScenarioScenarioExpectationArgs.builder()
.toolExpectation(EvaluationScenarioScenarioExpectationToolExpectationArgs.builder()
.expectedToolCall(EvaluationScenarioScenarioExpectationToolExpectationExpectedToolCallArgs.builder()
.id("tool-call-id")
.toolsetTool(EvaluationScenarioScenarioExpectationToolExpectationExpectedToolCallToolsetToolArgs.builder()
.toolset(Output.tuple(app.project(), app.appId(), toolset.toolsetId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
var toolsetId = values.t3;
return String.format("projects/%s/locations/us/apps/%s/toolsets/%s", project,appId,toolsetId);
}))
.toolId("dummy-tool")
.build())
.args(Map.of("param", "value"))
.build())
.mockToolResponse(EvaluationScenarioScenarioExpectationToolExpectationMockToolResponseArgs.builder()
.id("tool-call-id")
.response(Map.of("result", "mocked"))
.toolsetTool(EvaluationScenarioScenarioExpectationToolExpectationMockToolResponseToolsetToolArgs.builder()
.toolset(Output.tuple(app.project(), app.appId(), toolset.toolsetId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
var toolsetId = values.t3;
return String.format("projects/%s/locations/us/apps/%s/toolsets/%s", project,appId,toolsetId);
}))
.toolId("dummy-tool")
.build())
.build())
.build())
.build(),
EvaluationScenarioScenarioExpectationArgs.builder()
.agentResponse(EvaluationScenarioScenarioExpectationAgentResponseArgs.builder()
.role("agent")
.chunks(
EvaluationScenarioScenarioExpectationAgentResponseChunkArgs.builder()
.text("Hello")
.build(),
EvaluationScenarioScenarioExpectationAgentResponseChunkArgs.builder()
.toolCall(EvaluationScenarioScenarioExpectationAgentResponseChunkToolCallArgs.builder()
.id("tool-call-id-3")
.toolsetTool(EvaluationScenarioScenarioExpectationAgentResponseChunkToolCallToolsetToolArgs.builder()
.toolset(Output.tuple(app.project(), app.appId(), toolset.toolsetId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
var toolsetId = values.t3;
return String.format("projects/%s/locations/us/apps/%s/toolsets/%s", project,appId,toolsetId);
}))
.toolId("dummy-tool")
.build())
.args(Map.of("param", "value"))
.build())
.build(),
EvaluationScenarioScenarioExpectationAgentResponseChunkArgs.builder()
.toolResponse(EvaluationScenarioScenarioExpectationAgentResponseChunkToolResponseArgs.builder()
.id("tool-call-id-3")
.response(Map.of("result", "success"))
.toolsetTool(EvaluationScenarioScenarioExpectationAgentResponseChunkToolResponseToolsetToolArgs.builder()
.toolset(Output.tuple(app.project(), app.appId(), toolset.toolsetId()).applyValue(values -> {
var project = values.t1;
var appId = values.t2;
var toolsetId = values.t3;
return String.format("projects/%s/locations/us/apps/%s/toolsets/%s", project,appId,toolsetId);
}))
.toolId("dummy-tool")
.build())
.build())
.build())
.build())
.build())
.build())
.build());
}
}
resources:
app:
type: gcp:ces:App
properties:
appId: app-id-scenario-ts
location: us
displayName: my-app-scenario-ts
languageSettings:
defaultLanguageCode: en-US
timeZoneSettings:
timeZone: America/Los_Angeles
toolset:
type: gcp:ces:Toolset
properties:
toolsetId: ts-scen
location: us
app: ${app.appId}
displayName: Basic toolset display name
description: Test description
executionType: SYNCHRONOUS
openApiToolset:
openApiSchema: |
openapi: 3.0.0
info:
title: My Sample API
version: 1.0.0
description: A simple API example
servers:
- url: https://api.example.com/v1
paths: {}
ignoreUnknownFields: false
cesEvaluationScenarioToolset:
type: gcp:ces:Evaluation
name: ces_evaluation_scenario_toolset
properties:
evaluationId: eval-scen-ts
displayName: my-evaluation-scenario-toolset
location: us
app: ${app.appId}
description: Full evaluation for testing scenario with toolset
tags:
- test
- full
- scenario
- toolset
scenario:
task: Test task
maxTurns: 5
rubrics:
- projects/${app.project}/locations/us/apps/${app.appId}/rubrics/dummy-rubric
userGoalBehavior: USER_GOAL_SATISFIED
taskCompletionBehavior: TASK_SATISFIED
variableOverrides:
key: value
evaluationExpectations:
- projects/${app.project}/locations/us/apps/${app.appId}/evaluationExpectations/dummy-exp
userFacts:
- name: user_name
value: John Doe
scenarioExpectations:
- toolExpectation:
expectedToolCall:
id: tool-call-id
toolsetTool:
toolset: projects/${app.project}/locations/us/apps/${app.appId}/toolsets/${toolset.toolsetId}
toolId: dummy-tool
args:
param: value
mockToolResponse:
id: tool-call-id
response:
result: mocked
toolsetTool:
toolset: projects/${app.project}/locations/us/apps/${app.appId}/toolsets/${toolset.toolsetId}
toolId: dummy-tool
- agentResponse:
role: agent
chunks:
- text: Hello
- toolCall:
id: tool-call-id-3
toolsetTool:
toolset: projects/${app.project}/locations/us/apps/${app.appId}/toolsets/${toolset.toolsetId}
toolId: dummy-tool
args:
param: value
- toolResponse:
id: tool-call-id-3
response:
result: success
toolsetTool:
toolset: projects/${app.project}/locations/us/apps/${app.appId}/toolsets/${toolset.toolsetId}
toolId: dummy-tool
Example coming soon!
Create Evaluation Resource
Resources are created with functions called constructors. To learn more about declaring and configuring resources, see Resources.
Constructor syntax
new Evaluation(name: string, args: EvaluationArgs, opts?: CustomResourceOptions);@overload
def Evaluation(resource_name: str,
args: EvaluationArgs,
opts: Optional[ResourceOptions] = None)
@overload
def Evaluation(resource_name: str,
opts: Optional[ResourceOptions] = None,
app: Optional[str] = None,
display_name: Optional[str] = None,
evaluation_id: Optional[str] = None,
location: Optional[str] = None,
deletion_policy: Optional[str] = None,
description: Optional[str] = None,
golden: Optional[EvaluationGoldenArgs] = None,
project: Optional[str] = None,
scenario: Optional[EvaluationScenarioArgs] = None,
tags: Optional[Sequence[str]] = None)func NewEvaluation(ctx *Context, name string, args EvaluationArgs, opts ...ResourceOption) (*Evaluation, error)public Evaluation(string name, EvaluationArgs args, CustomResourceOptions? opts = null)
public Evaluation(String name, EvaluationArgs args)
public Evaluation(String name, EvaluationArgs args, CustomResourceOptions options)
type: gcp:ces:Evaluation
properties: # The arguments to resource properties.
options: # Bag of options to control resource's behavior.
resource "gcp_ces_evaluation" "name" {
# resource properties
}Parameters
- name string
- The unique name of the resource.
- args EvaluationArgs
- The arguments to resource properties.
- opts CustomResourceOptions
- Bag of options to control resource's behavior.
- resource_name str
- The unique name of the resource.
- args EvaluationArgs
- The arguments to resource properties.
- opts ResourceOptions
- Bag of options to control resource's behavior.
- ctx Context
- Context object for the current deployment.
- name string
- The unique name of the resource.
- args EvaluationArgs
- The arguments to resource properties.
- opts ResourceOption
- Bag of options to control resource's behavior.
- name string
- The unique name of the resource.
- args EvaluationArgs
- The arguments to resource properties.
- opts CustomResourceOptions
- Bag of options to control resource's behavior.
- name String
- The unique name of the resource.
- args EvaluationArgs
- The arguments to resource properties.
- options CustomResourceOptions
- Bag of options to control resource's behavior.
Constructor example
The following reference example uses placeholder values for all input properties.
var evaluationResource = new Gcp.Ces.Evaluation("evaluationResource", new()
{
App = "string",
DisplayName = "string",
EvaluationId = "string",
Location = "string",
DeletionPolicy = "string",
Description = "string",
Golden = new Gcp.Ces.Inputs.EvaluationGoldenArgs
{
Turns = new[]
{
new Gcp.Ces.Inputs.EvaluationGoldenTurnArgs
{
Steps = new[]
{
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepArgs
{
AgentTransfer = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepAgentTransferArgs
{
TargetAgent = "string",
DisplayName = "string",
},
Expectation = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationArgs
{
AgentResponse = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseArgs
{
Chunks = new[]
{
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs
{
AgentTransfer = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkAgentTransferArgs
{
TargetAgent = "string",
DisplayName = "string",
},
Blob = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkBlobArgs
{
Data = "string",
MimeType = "string",
},
Image = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkImageArgs
{
Data = "string",
MimeType = "string",
},
Text = "string",
ToolCall = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallArgs
{
Args =
{
{ "string", "string" },
},
DisplayName = "string",
Id = "string",
Tool = "string",
ToolsetTool = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallToolsetToolArgs
{
Toolset = "string",
ToolId = "string",
},
},
ToolResponse = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseArgs
{
DisplayName = "string",
Id = "string",
Response =
{
{ "string", "string" },
},
Tool = "string",
ToolsetTool = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseToolsetToolArgs
{
Toolset = "string",
ToolId = "string",
},
},
UpdatedVariables =
{
{ "string", "string" },
},
},
},
Role = "string",
},
AgentTransfer = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationAgentTransferArgs
{
DisplayName = "string",
TargetAgent = "string",
},
MockToolResponse = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationMockToolResponseArgs
{
DisplayName = "string",
Id = "string",
Response =
{
{ "string", "string" },
},
Tool = "string",
ToolsetTool = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationMockToolResponseToolsetToolArgs
{
Toolset = "string",
ToolId = "string",
},
},
Note = "string",
ToolCall = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationToolCallArgs
{
Args =
{
{ "string", "string" },
},
DisplayName = "string",
Id = "string",
Tool = "string",
ToolsetTool = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationToolCallToolsetToolArgs
{
Toolset = "string",
ToolId = "string",
},
},
ToolResponse = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationToolResponseArgs
{
DisplayName = "string",
Id = "string",
Response =
{
{ "string", "string" },
},
Tool = "string",
ToolsetTool = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationToolResponseToolsetToolArgs
{
Toolset = "string",
ToolId = "string",
},
},
UpdatedVariables = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepExpectationUpdatedVariablesArgs
{
Notes = "string",
},
},
UserInput = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputArgs
{
Audio = "string",
Blob = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputBlobArgs
{
Data = "string",
MimeType = "string",
},
Dtmf = "string",
Event = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputEventArgs
{
Event = "string",
},
Image = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputImageArgs
{
Data = "string",
MimeType = "string",
},
Text = "string",
ToolResponses = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputToolResponsesArgs
{
ToolResponses = new[]
{
new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputToolResponsesToolResponseArgs
{
Response =
{
{ "string", "string" },
},
DisplayName = "string",
Id = "string",
Tool = "string",
ToolsetTool = new Gcp.Ces.Inputs.EvaluationGoldenTurnStepUserInputToolResponsesToolResponseToolsetToolArgs
{
Toolset = "string",
ToolId = "string",
},
},
},
},
Variables =
{
{ "string", "string" },
},
WillContinue = false,
},
},
},
RootSpans = new[]
{
new Gcp.Ces.Inputs.EvaluationGoldenTurnRootSpanArgs
{
Attributes = "string",
ChildSpans = "string",
Duration = "string",
EndTime = "string",
Name = "string",
StartTime = "string",
},
},
},
},
EvaluationExpectations = new[]
{
"string",
},
},
Project = "string",
Scenario = new Gcp.Ces.Inputs.EvaluationScenarioArgs
{
Rubrics = new[]
{
"string",
},
ScenarioExpectations = new[]
{
new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationArgs
{
AgentResponse = new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationAgentResponseArgs
{
Chunks = new[]
{
new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkArgs
{
AgentTransfer = new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkAgentTransferArgs
{
TargetAgent = "string",
DisplayName = "string",
},
Blob = new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkBlobArgs
{
Data = "string",
MimeType = "string",
},
Image = new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkImageArgs
{
Data = "string",
MimeType = "string",
},
Text = "string",
ToolCall = new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkToolCallArgs
{
Args =
{
{ "string", "string" },
},
DisplayName = "string",
Id = "string",
Tool = "string",
ToolsetTool = new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkToolCallToolsetToolArgs
{
Toolset = "string",
ToolId = "string",
},
},
ToolResponse = new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkToolResponseArgs
{
DisplayName = "string",
Id = "string",
Response =
{
{ "string", "string" },
},
Tool = "string",
ToolsetTool = new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationAgentResponseChunkToolResponseToolsetToolArgs
{
Toolset = "string",
ToolId = "string",
},
},
UpdatedVariables =
{
{ "string", "string" },
},
},
},
Role = "string",
},
ToolExpectation = new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationToolExpectationArgs
{
ExpectedToolCall = new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationToolExpectationExpectedToolCallArgs
{
Args =
{
{ "string", "string" },
},
DisplayName = "string",
Id = "string",
Tool = "string",
ToolsetTool = new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationToolExpectationExpectedToolCallToolsetToolArgs
{
ToolId = "string",
Toolset = "string",
},
},
MockToolResponse = new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationToolExpectationMockToolResponseArgs
{
DisplayName = "string",
Id = "string",
Response =
{
{ "string", "string" },
},
Tool = "string",
ToolsetTool = new Gcp.Ces.Inputs.EvaluationScenarioScenarioExpectationToolExpectationMockToolResponseToolsetToolArgs
{
ToolId = "string",
Toolset = "string",
},
},
},
},
},
Task = "string",
EvaluationExpectations = new[]
{
"string",
},
MaxTurns = 0,
TaskCompletionBehavior = "string",
UserFacts = new[]
{
new Gcp.Ces.Inputs.EvaluationScenarioUserFactArgs
{
Name = "string",
Value = "string",
},
},
UserGoalBehavior = "string",
VariableOverrides =
{
{ "string", "string" },
},
},
Tags = new[]
{
"string",
},
});
example, err := ces.NewEvaluation(ctx, "evaluationResource", &ces.EvaluationArgs{
App: pulumi.String("string"),
DisplayName: pulumi.String("string"),
EvaluationId: pulumi.String("string"),
Location: pulumi.String("string"),
DeletionPolicy: pulumi.String("string"),
Description: pulumi.String("string"),
Golden: &ces.EvaluationGoldenArgs{
Turns: ces.EvaluationGoldenTurnArray{
&ces.EvaluationGoldenTurnArgs{
Steps: ces.EvaluationGoldenTurnStepArray{
&ces.EvaluationGoldenTurnStepArgs{
AgentTransfer: &ces.EvaluationGoldenTurnStepAgentTransferArgs{
TargetAgent: pulumi.String("string"),
DisplayName: pulumi.String("string"),
},
Expectation: &ces.EvaluationGoldenTurnStepExpectationArgs{
AgentResponse: &ces.EvaluationGoldenTurnStepExpectationAgentResponseArgs{
Chunks: ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkArray{
&ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs{
AgentTransfer: &ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkAgentTransferArgs{
TargetAgent: pulumi.String("string"),
DisplayName: pulumi.String("string"),
},
Blob: &ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkBlobArgs{
Data: pulumi.String("string"),
MimeType: pulumi.String("string"),
},
Image: &ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkImageArgs{
Data: pulumi.String("string"),
MimeType: pulumi.String("string"),
},
Text: pulumi.String("string"),
ToolCall: &ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallArgs{
Args: pulumi.StringMap{
"string": pulumi.String("string"),
},
DisplayName: pulumi.String("string"),
Id: pulumi.String("string"),
Tool: pulumi.String("string"),
ToolsetTool: &ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallToolsetToolArgs{
Toolset: pulumi.String("string"),
ToolId: pulumi.String("string"),
},
},
ToolResponse: &ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseArgs{
DisplayName: pulumi.String("string"),
Id: pulumi.String("string"),
Response: pulumi.StringMap{
"string": pulumi.String("string"),
},
Tool: pulumi.String("string"),
ToolsetTool: &ces.EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseToolsetToolArgs{
Toolset: pulumi.String("string"),
ToolId: pulumi.String("string"),
},
},
UpdatedVariables: pulumi.StringMap{
"string": pulumi.String("string"),
},
},
},
Role: pulumi.String("string"),
},
AgentTransfer: &ces.EvaluationGoldenTurnStepExpectationAgentTransferArgs{
DisplayName: pulumi.String("string"),
TargetAgent: pulumi.String("string"),
},
MockToolResponse: &ces.EvaluationGoldenTurnStepExpectationMockToolResponseArgs{
DisplayName: pulumi.String("string"),
Id: pulumi.String("string"),
Response: pulumi.StringMap{
"string": pulumi.String("string"),
},
Tool: pulumi.String("string"),
ToolsetTool: &ces.EvaluationGoldenTurnStepExpectationMockToolResponseToolsetToolArgs{
Toolset: pulumi.String("string"),
ToolId: pulumi.String("string"),
},
},
Note: pulumi.String("string"),
ToolCall: &ces.EvaluationGoldenTurnStepExpectationToolCallArgs{
Args: pulumi.StringMap{
"string": pulumi.String("string"),
},
DisplayName: pulumi.String("string"),
Id: pulumi.String("string"),
Tool: pulumi.String("string"),
ToolsetTool: &ces.EvaluationGoldenTurnStepExpectationToolCallToolsetToolArgs{
Toolset: pulumi.String("string"),
ToolId: pulumi.String("string"),
},
},
ToolResponse: &ces.EvaluationGoldenTurnStepExpectationToolResponseArgs{
DisplayName: pulumi.String("string"),
Id: pulumi.String("string"),
Response: pulumi.StringMap{
"string": pulumi.String("string"),
},
Tool: pulumi.String("string"),
ToolsetTool: &ces.EvaluationGoldenTurnStepExpectationToolResponseToolsetToolArgs{
Toolset: pulumi.String("string"),
ToolId: pulumi.String("string"),
},
},
UpdatedVariables: &ces.EvaluationGoldenTurnStepExpectationUpdatedVariablesArgs{
Notes: pulumi.String("string"),
},
},
UserInput: &ces.EvaluationGoldenTurnStepUserInputArgs{
Audio: pulumi.String("string"),
Blob: &ces.EvaluationGoldenTurnStepUserInputBlobArgs{
Data: pulumi.String("string"),
MimeType: pulumi.String("string"),
},
Dtmf: pulumi.String("string"),
Event: &ces.EvaluationGoldenTurnStepUserInputEventArgs{
Event: pulumi.String("string"),
},
Image: &ces.EvaluationGoldenTurnStepUserInputImageArgs{
Data: pulumi.String("string"),
MimeType: pulumi.String("string"),
},
Text: pulumi.String("string"),
ToolResponses: &ces.EvaluationGoldenTurnStepUserInputToolResponsesArgs{
ToolResponses: ces.EvaluationGoldenTurnStepUserInputToolResponsesToolResponseArray{
&ces.EvaluationGoldenTurnStepUserInputToolResponsesToolResponseArgs{
Response: pulumi.StringMap{
"string": pulumi.String("string"),
},
DisplayName: pulumi.String("string"),
Id: pulumi.String("string"),
Tool: pulumi.String("string"),
ToolsetTool: &ces.EvaluationGoldenTurnStepUserInputToolResponsesToolResponseToolsetToolArgs{
Toolset: pulumi.String("string"),
ToolId: pulumi.String("string"),
},
},
},
},
Variables: pulumi.StringMap{
"string": pulumi.String("string"),
},
WillContinue: pulumi.Bool(false),
},
},
},
RootSpans: ces.EvaluationGoldenTurnRootSpanArray{
&ces.EvaluationGoldenTurnRootSpanArgs{
Attributes: pulumi.String("string"),
ChildSpans: pulumi.String("string"),
Duration: pulumi.String("string"),
EndTime: pulumi.String("string"),
Name: pulumi.String("string"),
StartTime: pulumi.String("string"),
},
},
},
},
EvaluationExpectations: pulumi.StringArray{
pulumi.String("string"),
},
},
Project: pulumi.String("string"),
Scenario: &ces.EvaluationScenarioArgs{
Rubrics: pulumi.StringArray{
pulumi.String("string"),
},
ScenarioExpectations: ces.EvaluationScenarioScenarioExpectationArray{
&ces.EvaluationScenarioScenarioExpectationArgs{
AgentResponse: &ces.EvaluationScenarioScenarioExpectationAgentResponseArgs{
Chunks: ces.EvaluationScenarioScenarioExpectationAgentResponseChunkArray{
&ces.EvaluationScenarioScenarioExpectationAgentResponseChunkArgs{
AgentTransfer: &ces.EvaluationScenarioScenarioExpectationAgentResponseChunkAgentTransferArgs{
TargetAgent: pulumi.String("string"),
DisplayName: pulumi.String("string"),
},
Blob: &ces.EvaluationScenarioScenarioExpectationAgentResponseChunkBlobArgs{
Data: pulumi.String("string"),
MimeType: pulumi.String("string"),
},
Image: &ces.EvaluationScenarioScenarioExpectationAgentResponseChunkImageArgs{
Data: pulumi.String("string"),
MimeType: pulumi.String("string"),
},
Text: pulumi.String("string"),
ToolCall: &ces.EvaluationScenarioScenarioExpectationAgentResponseChunkToolCallArgs{
Args: pulumi.StringMap{
"string": pulumi.String("string"),
},
DisplayName: pulumi.String("string"),
Id: pulumi.String("string"),
Tool: pulumi.String("string"),
ToolsetTool: &ces.EvaluationScenarioScenarioExpectationAgentResponseChunkToolCallToolsetToolArgs{
Toolset: pulumi.String("string"),
ToolId: pulumi.String("string"),
},
},
ToolResponse: &ces.EvaluationScenarioScenarioExpectationAgentResponseChunkToolResponseArgs{
DisplayName: pulumi.String("string"),
Id: pulumi.String("string"),
Response: pulumi.StringMap{
"string": pulumi.String("string"),
},
Tool: pulumi.String("string"),
ToolsetTool: &ces.EvaluationScenarioScenarioExpectationAgentResponseChunkToolResponseToolsetToolArgs{
Toolset: pulumi.String("string"),
ToolId: pulumi.String("string"),
},
},
UpdatedVariables: pulumi.StringMap{
"string": pulumi.String("string"),
},
},
},
Role: pulumi.String("string"),
},
ToolExpectation: &ces.EvaluationScenarioScenarioExpectationToolExpectationArgs{
ExpectedToolCall: &ces.EvaluationScenarioScenarioExpectationToolExpectationExpectedToolCallArgs{
Args: pulumi.StringMap{
"string": pulumi.String("string"),
},
DisplayName: pulumi.String("string"),
Id: pulumi.String("string"),
Tool: pulumi.String("string"),
ToolsetTool: &ces.EvaluationScenarioScenarioExpectationToolExpectationExpectedToolCallToolsetToolArgs{
ToolId: pulumi.String("string"),
Toolset: pulumi.String("string"),
},
},
MockToolResponse: &ces.EvaluationScenarioScenarioExpectationToolExpectationMockToolResponseArgs{
DisplayName: pulumi.String("string"),
Id: pulumi.String("string"),
Response: pulumi.StringMap{
"string": pulumi.String("string"),
},
Tool: pulumi.String("string"),
ToolsetTool: &ces.EvaluationScenarioScenarioExpectationToolExpectationMockToolResponseToolsetToolArgs{
ToolId: pulumi.String("string"),
Toolset: pulumi.String("string"),
},
},
},
},
},
Task: pulumi.String("string"),
EvaluationExpectations: pulumi.StringArray{
pulumi.String("string"),
},
MaxTurns: pulumi.Int(0),
TaskCompletionBehavior: pulumi.String("string"),
UserFacts: ces.EvaluationScenarioUserFactArray{
&ces.EvaluationScenarioUserFactArgs{
Name: pulumi.String("string"),
Value: pulumi.String("string"),
},
},
UserGoalBehavior: pulumi.String("string"),
VariableOverrides: pulumi.StringMap{
"string": pulumi.String("string"),
},
},
Tags: pulumi.StringArray{
pulumi.String("string"),
},
})
resource "gcp_ces_evaluation" "evaluationResource" {
app = "string"
display_name = "string"
evaluation_id = "string"
location = "string"
deletion_policy = "string"
description = "string"
golden = {
turns = [{
"steps" = [{
"agentTransfer" = {
"targetAgent" = "string"
"displayName" = "string"
}
"expectation" = {
"agentResponse" = {
"chunks" = [{
"agentTransfer" = {
"targetAgent" = "string"
"displayName" = "string"
}
"blob" = {
"data" = "string"
"mimeType" = "string"
}
"image" = {
"data" = "string"
"mimeType" = "string"
}
"text" = "string"
"toolCall" = {
"args" = {
"string" = "string"
}
"displayName" = "string"
"id" = "string"
"tool" = "string"
"toolsetTool" = {
"toolset" = "string"
"toolId" = "string"
}
}
"toolResponse" = {
"displayName" = "string"
"id" = "string"
"response" = {
"string" = "string"
}
"tool" = "string"
"toolsetTool" = {
"toolset" = "string"
"toolId" = "string"
}
}
"updatedVariables" = {
"string" = "string"
}
}]
"role" = "string"
}
"agentTransfer" = {
"displayName" = "string"
"targetAgent" = "string"
}
"mockToolResponse" = {
"displayName" = "string"
"id" = "string"
"response" = {
"string" = "string"
}
"tool" = "string"
"toolsetTool" = {
"toolset" = "string"
"toolId" = "string"
}
}
"note" = "string"
"toolCall" = {
"args" = {
"string" = "string"
}
"displayName" = "string"
"id" = "string"
"tool" = "string"
"toolsetTool" = {
"toolset" = "string"
"toolId" = "string"
}
}
"toolResponse" = {
"displayName" = "string"
"id" = "string"
"response" = {
"string" = "string"
}
"tool" = "string"
"toolsetTool" = {
"toolset" = "string"
"toolId" = "string"
}
}
"updatedVariables" = {
"notes" = "string"
}
}
"userInput" = {
"audio" = "string"
"blob" = {
"data" = "string"
"mimeType" = "string"
}
"dtmf" = "string"
"event" = {
"event" = "string"
}
"image" = {
"data" = "string"
"mimeType" = "string"
}
"text" = "string"
"toolResponses" = {
"toolResponses" = [{
"response" = {
"string" = "string"
}
"displayName" = "string"
"id" = "string"
"tool" = "string"
"toolsetTool" = {
"toolset" = "string"
"toolId" = "string"
}
}]
}
"variables" = {
"string" = "string"
}
"willContinue" = false
}
}]
"rootSpans" = [{
"attributes" = "string"
"childSpans" = "string"
"duration" = "string"
"endTime" = "string"
"name" = "string"
"startTime" = "string"
}]
}]
evaluation_expectations = ["string"]
}
project = "string"
scenario = {
rubrics = ["string"]
scenario_expectations = [{
"agentResponse" = {
"chunks" = [{
"agentTransfer" = {
"targetAgent" = "string"
"displayName" = "string"
}
"blob" = {
"data" = "string"
"mimeType" = "string"
}
"image" = {
"data" = "string"
"mimeType" = "string"
}
"text" = "string"
"toolCall" = {
"args" = {
"string" = "string"
}
"displayName" = "string"
"id" = "string"
"tool" = "string"
"toolsetTool" = {
"toolset" = "string"
"toolId" = "string"
}
}
"toolResponse" = {
"displayName" = "string"
"id" = "string"
"response" = {
"string" = "string"
}
"tool" = "string"
"toolsetTool" = {
"toolset" = "string"
"toolId" = "string"
}
}
"updatedVariables" = {
"string" = "string"
}
}]
"role" = "string"
}
"toolExpectation" = {
"expectedToolCall" = {
"args" = {
"string" = "string"
}
"displayName" = "string"
"id" = "string"
"tool" = "string"
"toolsetTool" = {
"toolId" = "string"
"toolset" = "string"
}
}
"mockToolResponse" = {
"displayName" = "string"
"id" = "string"
"response" = {
"string" = "string"
}
"tool" = "string"
"toolsetTool" = {
"toolId" = "string"
"toolset" = "string"
}
}
}
}]
task = "string"
evaluation_expectations = ["string"]
max_turns = 0
task_completion_behavior = "string"
user_facts = [{
"name" = "string"
"value" = "string"
}]
user_goal_behavior = "string"
variable_overrides = {
"string" = "string"
}
}
tags = ["string"]
}
var evaluationResource = new Evaluation("evaluationResource", EvaluationArgs.builder()
.app("string")
.displayName("string")
.evaluationId("string")
.location("string")
.deletionPolicy("string")
.description("string")
.golden(EvaluationGoldenArgs.builder()
.turns(EvaluationGoldenTurnArgs.builder()
.steps(EvaluationGoldenTurnStepArgs.builder()
.agentTransfer(EvaluationGoldenTurnStepAgentTransferArgs.builder()
.targetAgent("string")
.displayName("string")
.build())
.expectation(EvaluationGoldenTurnStepExpectationArgs.builder()
.agentResponse(EvaluationGoldenTurnStepExpectationAgentResponseArgs.builder()
.chunks(EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs.builder()
.agentTransfer(EvaluationGoldenTurnStepExpectationAgentResponseChunkAgentTransferArgs.builder()
.targetAgent("string")
.displayName("string")
.build())
.blob(EvaluationGoldenTurnStepExpectationAgentResponseChunkBlobArgs.builder()
.data("string")
.mimeType("string")
.build())
.image(EvaluationGoldenTurnStepExpectationAgentResponseChunkImageArgs.builder()
.data("string")
.mimeType("string")
.build())
.text("string")
.toolCall(EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallArgs.builder()
.args(Map.of("string", "string"))
.displayName("string")
.id("string")
.tool("string")
.toolsetTool(EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallToolsetToolArgs.builder()
.toolset("string")
.toolId("string")
.build())
.build())
.toolResponse(EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseArgs.builder()
.displayName("string")
.id("string")
.response(Map.of("string", "string"))
.tool("string")
.toolsetTool(EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseToolsetToolArgs.builder()
.toolset("string")
.toolId("string")
.build())
.build())
.updatedVariables(Map.of("string", "string"))
.build())
.role("string")
.build())
.agentTransfer(EvaluationGoldenTurnStepExpectationAgentTransferArgs.builder()
.displayName("string")
.targetAgent("string")
.build())
.mockToolResponse(EvaluationGoldenTurnStepExpectationMockToolResponseArgs.builder()
.displayName("string")
.id("string")
.response(Map.of("string", "string"))
.tool("string")
.toolsetTool(EvaluationGoldenTurnStepExpectationMockToolResponseToolsetToolArgs.builder()
.toolset("string")
.toolId("string")
.build())
.build())
.note("string")
.toolCall(EvaluationGoldenTurnStepExpectationToolCallArgs.builder()
.args(Map.of("string", "string"))
.displayName("string")
.id("string")
.tool("string")
.toolsetTool(EvaluationGoldenTurnStepExpectationToolCallToolsetToolArgs.builder()
.toolset("string")
.toolId("string")
.build())
.build())
.toolResponse(EvaluationGoldenTurnStepExpectationToolResponseArgs.builder()
.displayName("string")
.id("string")
.response(Map.of("string", "string"))
.tool("string")
.toolsetTool(EvaluationGoldenTurnStepExpectationToolResponseToolsetToolArgs.builder()
.toolset("string")
.toolId("string")
.build())
.build())
.updatedVariables(EvaluationGoldenTurnStepExpectationUpdatedVariablesArgs.builder()
.notes("string")
.build())
.build())
.userInput(EvaluationGoldenTurnStepUserInputArgs.builder()
.audio("string")
.blob(EvaluationGoldenTurnStepUserInputBlobArgs.builder()
.data("string")
.mimeType("string")
.build())
.dtmf("string")
.event(EvaluationGoldenTurnStepUserInputEventArgs.builder()
.event("string")
.build())
.image(EvaluationGoldenTurnStepUserInputImageArgs.builder()
.data("string")
.mimeType("string")
.build())
.text("string")
.toolResponses(EvaluationGoldenTurnStepUserInputToolResponsesArgs.builder()
.toolResponses(EvaluationGoldenTurnStepUserInputToolResponsesToolResponseArgs.builder()
.response(Map.of("string", "string"))
.displayName("string")
.id("string")
.tool("string")
.toolsetTool(EvaluationGoldenTurnStepUserInputToolResponsesToolResponseToolsetToolArgs.builder()
.toolset("string")
.toolId("string")
.build())
.build())
.build())
.variables(Map.of("string", "string"))
.willContinue(false)
.build())
.build())
.rootSpans(EvaluationGoldenTurnRootSpanArgs.builder()
.attributes("string")
.childSpans("string")
.duration("string")
.endTime("string")
.name("string")
.startTime("string")
.build())
.build())
.evaluationExpectations("string")
.build())
.project("string")
.scenario(EvaluationScenarioArgs.builder()
.rubrics("string")
.scenarioExpectations(EvaluationScenarioScenarioExpectationArgs.builder()
.agentResponse(EvaluationScenarioScenarioExpectationAgentResponseArgs.builder()
.chunks(EvaluationScenarioScenarioExpectationAgentResponseChunkArgs.builder()
.agentTransfer(EvaluationScenarioScenarioExpectationAgentResponseChunkAgentTransferArgs.builder()
.targetAgent("string")
.displayName("string")
.build())
.blob(EvaluationScenarioScenarioExpectationAgentResponseChunkBlobArgs.builder()
.data("string")
.mimeType("string")
.build())
.image(EvaluationScenarioScenarioExpectationAgentResponseChunkImageArgs.builder()
.data("string")
.mimeType("string")
.build())
.text("string")
.toolCall(EvaluationScenarioScenarioExpectationAgentResponseChunkToolCallArgs.builder()
.args(Map.of("string", "string"))
.displayName("string")
.id("string")
.tool("string")
.toolsetTool(EvaluationScenarioScenarioExpectationAgentResponseChunkToolCallToolsetToolArgs.builder()
.toolset("string")
.toolId("string")
.build())
.build())
.toolResponse(EvaluationScenarioScenarioExpectationAgentResponseChunkToolResponseArgs.builder()
.displayName("string")
.id("string")
.response(Map.of("string", "string"))
.tool("string")
.toolsetTool(EvaluationScenarioScenarioExpectationAgentResponseChunkToolResponseToolsetToolArgs.builder()
.toolset("string")
.toolId("string")
.build())
.build())
.updatedVariables(Map.of("string", "string"))
.build())
.role("string")
.build())
.toolExpectation(EvaluationScenarioScenarioExpectationToolExpectationArgs.builder()
.expectedToolCall(EvaluationScenarioScenarioExpectationToolExpectationExpectedToolCallArgs.builder()
.args(Map.of("string", "string"))
.displayName("string")
.id("string")
.tool("string")
.toolsetTool(EvaluationScenarioScenarioExpectationToolExpectationExpectedToolCallToolsetToolArgs.builder()
.toolId("string")
.toolset("string")
.build())
.build())
.mockToolResponse(EvaluationScenarioScenarioExpectationToolExpectationMockToolResponseArgs.builder()
.displayName("string")
.id("string")
.response(Map.of("string", "string"))
.tool("string")
.toolsetTool(EvaluationScenarioScenarioExpectationToolExpectationMockToolResponseToolsetToolArgs.builder()
.toolId("string")
.toolset("string")
.build())
.build())
.build())
.build())
.task("string")
.evaluationExpectations("string")
.maxTurns(0)
.taskCompletionBehavior("string")
.userFacts(EvaluationScenarioUserFactArgs.builder()
.name("string")
.value("string")
.build())
.userGoalBehavior("string")
.variableOverrides(Map.of("string", "string"))
.build())
.tags("string")
.build());
evaluation_resource = gcp.ces.Evaluation("evaluationResource",
app="string",
display_name="string",
evaluation_id="string",
location="string",
deletion_policy="string",
description="string",
golden={
"turns": [{
"steps": [{
"agent_transfer": {
"target_agent": "string",
"display_name": "string",
},
"expectation": {
"agent_response": {
"chunks": [{
"agent_transfer": {
"target_agent": "string",
"display_name": "string",
},
"blob": {
"data": "string",
"mime_type": "string",
},
"image": {
"data": "string",
"mime_type": "string",
},
"text": "string",
"tool_call": {
"args": {
"string": "string",
},
"display_name": "string",
"id": "string",
"tool": "string",
"toolset_tool": {
"toolset": "string",
"tool_id": "string",
},
},
"tool_response": {
"display_name": "string",
"id": "string",
"response": {
"string": "string",
},
"tool": "string",
"toolset_tool": {
"toolset": "string",
"tool_id": "string",
},
},
"updated_variables": {
"string": "string",
},
}],
"role": "string",
},
"agent_transfer": {
"display_name": "string",
"target_agent": "string",
},
"mock_tool_response": {
"display_name": "string",
"id": "string",
"response": {
"string": "string",
},
"tool": "string",
"toolset_tool": {
"toolset": "string",
"tool_id": "string",
},
},
"note": "string",
"tool_call": {
"args": {
"string": "string",
},
"display_name": "string",
"id": "string",
"tool": "string",
"toolset_tool": {
"toolset": "string",
"tool_id": "string",
},
},
"tool_response": {
"display_name": "string",
"id": "string",
"response": {
"string": "string",
},
"tool": "string",
"toolset_tool": {
"toolset": "string",
"tool_id": "string",
},
},
"updated_variables": {
"notes": "string",
},
},
"user_input": {
"audio": "string",
"blob": {
"data": "string",
"mime_type": "string",
},
"dtmf": "string",
"event": {
"event": "string",
},
"image": {
"data": "string",
"mime_type": "string",
},
"text": "string",
"tool_responses": {
"tool_responses": [{
"response": {
"string": "string",
},
"display_name": "string",
"id": "string",
"tool": "string",
"toolset_tool": {
"toolset": "string",
"tool_id": "string",
},
}],
},
"variables": {
"string": "string",
},
"will_continue": False,
},
}],
"root_spans": [{
"attributes": "string",
"child_spans": "string",
"duration": "string",
"end_time": "string",
"name": "string",
"start_time": "string",
}],
}],
"evaluation_expectations": ["string"],
},
project="string",
scenario={
"rubrics": ["string"],
"scenario_expectations": [{
"agent_response": {
"chunks": [{
"agent_transfer": {
"target_agent": "string",
"display_name": "string",
},
"blob": {
"data": "string",
"mime_type": "string",
},
"image": {
"data": "string",
"mime_type": "string",
},
"text": "string",
"tool_call": {
"args": {
"string": "string",
},
"display_name": "string",
"id": "string",
"tool": "string",
"toolset_tool": {
"toolset": "string",
"tool_id": "string",
},
},
"tool_response": {
"display_name": "string",
"id": "string",
"response": {
"string": "string",
},
"tool": "string",
"toolset_tool": {
"toolset": "string",
"tool_id": "string",
},
},
"updated_variables": {
"string": "string",
},
}],
"role": "string",
},
"tool_expectation": {
"expected_tool_call": {
"args": {
"string": "string",
},
"display_name": "string",
"id": "string",
"tool": "string",
"toolset_tool": {
"tool_id": "string",
"toolset": "string",
},
},
"mock_tool_response": {
"display_name": "string",
"id": "string",
"response": {
"string": "string",
},
"tool": "string",
"toolset_tool": {
"tool_id": "string",
"toolset": "string",
},
},
},
}],
"task": "string",
"evaluation_expectations": ["string"],
"max_turns": 0,
"task_completion_behavior": "string",
"user_facts": [{
"name": "string",
"value": "string",
}],
"user_goal_behavior": "string",
"variable_overrides": {
"string": "string",
},
},
tags=["string"])
const evaluationResource = new gcp.ces.Evaluation("evaluationResource", {
app: "string",
displayName: "string",
evaluationId: "string",
location: "string",
deletionPolicy: "string",
description: "string",
golden: {
turns: [{
steps: [{
agentTransfer: {
targetAgent: "string",
displayName: "string",
},
expectation: {
agentResponse: {
chunks: [{
agentTransfer: {
targetAgent: "string",
displayName: "string",
},
blob: {
data: "string",
mimeType: "string",
},
image: {
data: "string",
mimeType: "string",
},
text: "string",
toolCall: {
args: {
string: "string",
},
displayName: "string",
id: "string",
tool: "string",
toolsetTool: {
toolset: "string",
toolId: "string",
},
},
toolResponse: {
displayName: "string",
id: "string",
response: {
string: "string",
},
tool: "string",
toolsetTool: {
toolset: "string",
toolId: "string",
},
},
updatedVariables: {
string: "string",
},
}],
role: "string",
},
agentTransfer: {
displayName: "string",
targetAgent: "string",
},
mockToolResponse: {
displayName: "string",
id: "string",
response: {
string: "string",
},
tool: "string",
toolsetTool: {
toolset: "string",
toolId: "string",
},
},
note: "string",
toolCall: {
args: {
string: "string",
},
displayName: "string",
id: "string",
tool: "string",
toolsetTool: {
toolset: "string",
toolId: "string",
},
},
toolResponse: {
displayName: "string",
id: "string",
response: {
string: "string",
},
tool: "string",
toolsetTool: {
toolset: "string",
toolId: "string",
},
},
updatedVariables: {
notes: "string",
},
},
userInput: {
audio: "string",
blob: {
data: "string",
mimeType: "string",
},
dtmf: "string",
event: {
event: "string",
},
image: {
data: "string",
mimeType: "string",
},
text: "string",
toolResponses: {
toolResponses: [{
response: {
string: "string",
},
displayName: "string",
id: "string",
tool: "string",
toolsetTool: {
toolset: "string",
toolId: "string",
},
}],
},
variables: {
string: "string",
},
willContinue: false,
},
}],
rootSpans: [{
attributes: "string",
childSpans: "string",
duration: "string",
endTime: "string",
name: "string",
startTime: "string",
}],
}],
evaluationExpectations: ["string"],
},
project: "string",
scenario: {
rubrics: ["string"],
scenarioExpectations: [{
agentResponse: {
chunks: [{
agentTransfer: {
targetAgent: "string",
displayName: "string",
},
blob: {
data: "string",
mimeType: "string",
},
image: {
data: "string",
mimeType: "string",
},
text: "string",
toolCall: {
args: {
string: "string",
},
displayName: "string",
id: "string",
tool: "string",
toolsetTool: {
toolset: "string",
toolId: "string",
},
},
toolResponse: {
displayName: "string",
id: "string",
response: {
string: "string",
},
tool: "string",
toolsetTool: {
toolset: "string",
toolId: "string",
},
},
updatedVariables: {
string: "string",
},
}],
role: "string",
},
toolExpectation: {
expectedToolCall: {
args: {
string: "string",
},
displayName: "string",
id: "string",
tool: "string",
toolsetTool: {
toolId: "string",
toolset: "string",
},
},
mockToolResponse: {
displayName: "string",
id: "string",
response: {
string: "string",
},
tool: "string",
toolsetTool: {
toolId: "string",
toolset: "string",
},
},
},
}],
task: "string",
evaluationExpectations: ["string"],
maxTurns: 0,
taskCompletionBehavior: "string",
userFacts: [{
name: "string",
value: "string",
}],
userGoalBehavior: "string",
variableOverrides: {
string: "string",
},
},
tags: ["string"],
});
type: gcp:ces:Evaluation
properties:
app: string
deletionPolicy: string
description: string
displayName: string
evaluationId: string
golden:
evaluationExpectations:
- string
turns:
- rootSpans:
- attributes: string
childSpans: string
duration: string
endTime: string
name: string
startTime: string
steps:
- agentTransfer:
displayName: string
targetAgent: string
expectation:
agentResponse:
chunks:
- agentTransfer:
displayName: string
targetAgent: string
blob:
data: string
mimeType: string
image:
data: string
mimeType: string
text: string
toolCall:
args:
string: string
displayName: string
id: string
tool: string
toolsetTool:
toolId: string
toolset: string
toolResponse:
displayName: string
id: string
response:
string: string
tool: string
toolsetTool:
toolId: string
toolset: string
updatedVariables:
string: string
role: string
agentTransfer:
displayName: string
targetAgent: string
mockToolResponse:
displayName: string
id: string
response:
string: string
tool: string
toolsetTool:
toolId: string
toolset: string
note: string
toolCall:
args:
string: string
displayName: string
id: string
tool: string
toolsetTool:
toolId: string
toolset: string
toolResponse:
displayName: string
id: string
response:
string: string
tool: string
toolsetTool:
toolId: string
toolset: string
updatedVariables:
notes: string
userInput:
audio: string
blob:
data: string
mimeType: string
dtmf: string
event:
event: string
image:
data: string
mimeType: string
text: string
toolResponses:
toolResponses:
- displayName: string
id: string
response:
string: string
tool: string
toolsetTool:
toolId: string
toolset: string
variables:
string: string
willContinue: false
location: string
project: string
scenario:
evaluationExpectations:
- string
maxTurns: 0
rubrics:
- string
scenarioExpectations:
- agentResponse:
chunks:
- agentTransfer:
displayName: string
targetAgent: string
blob:
data: string
mimeType: string
image:
data: string
mimeType: string
text: string
toolCall:
args:
string: string
displayName: string
id: string
tool: string
toolsetTool:
toolId: string
toolset: string
toolResponse:
displayName: string
id: string
response:
string: string
tool: string
toolsetTool:
toolId: string
toolset: string
updatedVariables:
string: string
role: string
toolExpectation:
expectedToolCall:
args:
string: string
displayName: string
id: string
tool: string
toolsetTool:
toolId: string
toolset: string
mockToolResponse:
displayName: string
id: string
response:
string: string
tool: string
toolsetTool:
toolId: string
toolset: string
task: string
taskCompletionBehavior: string
userFacts:
- name: string
value: string
userGoalBehavior: string
variableOverrides:
string: string
tags:
- string
Evaluation Resource Properties
To learn more about resource properties and how to use them, see Inputs and Outputs in the Architecture and Concepts docs.
Inputs
In Python, inputs that are objects can be passed either as argument classes or as dictionary literals.
The Evaluation resource accepts the following input properties:
- App string
- (Required)
- Display
Name string - User-defined display name of the evaluation.
- Evaluation
Id string - The ID to use for the evaluation, which will become the final component of the evaluation's resource name. If not provided, a unique ID will be automatically assigned for the evaluation.
- Location string
- (Required)
- Deletion
Policy string - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- Description string
- User-defined description of the evaluation.
- Golden
Evaluation
Golden - Golden input. Structure is documented below.
- Project string
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- Scenario
Evaluation
Scenario - Scenario input. Structure is documented below.
- List<string>
- User defined tags to categorize the evaluation.
- App string
- (Required)
- Display
Name string - User-defined display name of the evaluation.
- Evaluation
Id string - The ID to use for the evaluation, which will become the final component of the evaluation's resource name. If not provided, a unique ID will be automatically assigned for the evaluation.
- Location string
- (Required)
- Deletion
Policy string - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- Description string
- User-defined description of the evaluation.
- Golden
Evaluation
Golden Args - Golden input. Structure is documented below.
- Project string
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- Scenario
Evaluation
Scenario Args - Scenario input. Structure is documented below.
- []string
- User defined tags to categorize the evaluation.
- app string
- (Required)
- display_
name string - User-defined display name of the evaluation.
- evaluation_
id string - The ID to use for the evaluation, which will become the final component of the evaluation's resource name. If not provided, a unique ID will be automatically assigned for the evaluation.
- location string
- (Required)
- deletion_
policy string - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- description string
- User-defined description of the evaluation.
- golden object
- Golden input. Structure is documented below.
- project string
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- scenario object
- Scenario input. Structure is documented below.
- list(string)
- User defined tags to categorize the evaluation.
- app String
- (Required)
- display
Name String - User-defined display name of the evaluation.
- evaluation
Id String - The ID to use for the evaluation, which will become the final component of the evaluation's resource name. If not provided, a unique ID will be automatically assigned for the evaluation.
- location String
- (Required)
- deletion
Policy String - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- description String
- User-defined description of the evaluation.
- golden
Evaluation
Golden - Golden input. Structure is documented below.
- project String
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- scenario
Evaluation
Scenario - Scenario input. Structure is documented below.
- List<String>
- User defined tags to categorize the evaluation.
- app string
- (Required)
- display
Name string - User-defined display name of the evaluation.
- evaluation
Id string - The ID to use for the evaluation, which will become the final component of the evaluation's resource name. If not provided, a unique ID will be automatically assigned for the evaluation.
- location string
- (Required)
- deletion
Policy string - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- description string
- User-defined description of the evaluation.
- golden
Evaluation
Golden - Golden input. Structure is documented below.
- project string
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- scenario
Evaluation
Scenario - Scenario input. Structure is documented below.
- string[]
- User defined tags to categorize the evaluation.
- app str
- (Required)
- display_
name str - User-defined display name of the evaluation.
- evaluation_
id str - The ID to use for the evaluation, which will become the final component of the evaluation's resource name. If not provided, a unique ID will be automatically assigned for the evaluation.
- location str
- (Required)
- deletion_
policy str - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- description str
- User-defined description of the evaluation.
- golden
Evaluation
Golden Args - Golden input. Structure is documented below.
- project str
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- scenario
Evaluation
Scenario Args - Scenario input. Structure is documented below.
- Sequence[str]
- User defined tags to categorize the evaluation.
- app String
- (Required)
- display
Name String - User-defined display name of the evaluation.
- evaluation
Id String - The ID to use for the evaluation, which will become the final component of the evaluation's resource name. If not provided, a unique ID will be automatically assigned for the evaluation.
- location String
- (Required)
- deletion
Policy String - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- description String
- User-defined description of the evaluation.
- golden Property Map
- Golden input. Structure is documented below.
- project String
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- scenario Property Map
- Scenario input. Structure is documented below.
- List<String>
- User defined tags to categorize the evaluation.
Outputs
All input properties are implicitly available as output properties. Additionally, the Evaluation resource produces the following output properties:
- Create
Time string - Timestamp when the evaluation was created.
- Created
By string - The user who created the evaluation.
- Etag string
- Etag used to ensure the object hasn't changed during a read-modify-write operation.
- Evaluation
Datasets List<string> - List of evaluation datasets the evaluation belongs to.
- Evaluation
Runs List<string> - The evaluation runs for this evaluation.
- Id string
- The provider-assigned unique ID for this managed resource.
- Invalid bool
- Whether the evaluation is invalid.
- Last
Updated stringBy - The user who last updated the evaluation.
- Name string
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - Update
Time string - Timestamp when the evaluation was last updated.
- Create
Time string - Timestamp when the evaluation was created.
- Created
By string - The user who created the evaluation.
- Etag string
- Etag used to ensure the object hasn't changed during a read-modify-write operation.
- Evaluation
Datasets []string - List of evaluation datasets the evaluation belongs to.
- Evaluation
Runs []string - The evaluation runs for this evaluation.
- Id string
- The provider-assigned unique ID for this managed resource.
- Invalid bool
- Whether the evaluation is invalid.
- Last
Updated stringBy - The user who last updated the evaluation.
- Name string
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - Update
Time string - Timestamp when the evaluation was last updated.
- create_
time string - Timestamp when the evaluation was created.
- created_
by string - The user who created the evaluation.
- etag string
- Etag used to ensure the object hasn't changed during a read-modify-write operation.
- evaluation_
datasets list(string) - List of evaluation datasets the evaluation belongs to.
- evaluation_
runs list(string) - The evaluation runs for this evaluation.
- id string
- The provider-assigned unique ID for this managed resource.
- invalid bool
- Whether the evaluation is invalid.
- last_
updated_ stringby - The user who last updated the evaluation.
- name string
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - update_
time string - Timestamp when the evaluation was last updated.
- create
Time String - Timestamp when the evaluation was created.
- created
By String - The user who created the evaluation.
- etag String
- Etag used to ensure the object hasn't changed during a read-modify-write operation.
- evaluation
Datasets List<String> - List of evaluation datasets the evaluation belongs to.
- evaluation
Runs List<String> - The evaluation runs for this evaluation.
- id String
- The provider-assigned unique ID for this managed resource.
- invalid Boolean
- Whether the evaluation is invalid.
- last
Updated StringBy - The user who last updated the evaluation.
- name String
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - update
Time String - Timestamp when the evaluation was last updated.
- create
Time string - Timestamp when the evaluation was created.
- created
By string - The user who created the evaluation.
- etag string
- Etag used to ensure the object hasn't changed during a read-modify-write operation.
- evaluation
Datasets string[] - List of evaluation datasets the evaluation belongs to.
- evaluation
Runs string[] - The evaluation runs for this evaluation.
- id string
- The provider-assigned unique ID for this managed resource.
- invalid boolean
- Whether the evaluation is invalid.
- last
Updated stringBy - The user who last updated the evaluation.
- name string
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - update
Time string - Timestamp when the evaluation was last updated.
- create_
time str - Timestamp when the evaluation was created.
- created_
by str - The user who created the evaluation.
- etag str
- Etag used to ensure the object hasn't changed during a read-modify-write operation.
- evaluation_
datasets Sequence[str] - List of evaluation datasets the evaluation belongs to.
- evaluation_
runs Sequence[str] - The evaluation runs for this evaluation.
- id str
- The provider-assigned unique ID for this managed resource.
- invalid bool
- Whether the evaluation is invalid.
- last_
updated_ strby - The user who last updated the evaluation.
- name str
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - update_
time str - Timestamp when the evaluation was last updated.
- create
Time String - Timestamp when the evaluation was created.
- created
By String - The user who created the evaluation.
- etag String
- Etag used to ensure the object hasn't changed during a read-modify-write operation.
- evaluation
Datasets List<String> - List of evaluation datasets the evaluation belongs to.
- evaluation
Runs List<String> - The evaluation runs for this evaluation.
- id String
- The provider-assigned unique ID for this managed resource.
- invalid Boolean
- Whether the evaluation is invalid.
- last
Updated StringBy - The user who last updated the evaluation.
- name String
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - update
Time String - Timestamp when the evaluation was last updated.
Look up Existing Evaluation Resource
Get an existing Evaluation resource’s state with the given name, ID, and optional extra properties used to qualify the lookup.
public static get(name: string, id: Input<ID>, state?: EvaluationState, opts?: CustomResourceOptions): Evaluation@staticmethod
def get(resource_name: str,
id: str,
opts: Optional[ResourceOptions] = None,
app: Optional[str] = None,
create_time: Optional[str] = None,
created_by: Optional[str] = None,
deletion_policy: Optional[str] = None,
description: Optional[str] = None,
display_name: Optional[str] = None,
etag: Optional[str] = None,
evaluation_datasets: Optional[Sequence[str]] = None,
evaluation_id: Optional[str] = None,
evaluation_runs: Optional[Sequence[str]] = None,
golden: Optional[EvaluationGoldenArgs] = None,
invalid: Optional[bool] = None,
last_updated_by: Optional[str] = None,
location: Optional[str] = None,
name: Optional[str] = None,
project: Optional[str] = None,
scenario: Optional[EvaluationScenarioArgs] = None,
tags: Optional[Sequence[str]] = None,
update_time: Optional[str] = None) -> Evaluationfunc GetEvaluation(ctx *Context, name string, id IDInput, state *EvaluationState, opts ...ResourceOption) (*Evaluation, error)public static Evaluation Get(string name, Input<string> id, EvaluationState? state, CustomResourceOptions? opts = null)public static Evaluation get(String name, Output<String> id, EvaluationState state, CustomResourceOptions options)resources: _: type: gcp:ces:Evaluation get: id: ${id}import {
to = gcp_ces_evaluation.example
id = "${id}"
}
- name
- The unique name of the resulting resource.
- id
- The unique provider ID of the resource to lookup.
- state
- Any extra arguments used during the lookup.
- opts
- A bag of options that control this resource's behavior.
- resource_name
- The unique name of the resulting resource.
- id
- The unique provider ID of the resource to lookup.
- name
- The unique name of the resulting resource.
- id
- The unique provider ID of the resource to lookup.
- state
- Any extra arguments used during the lookup.
- opts
- A bag of options that control this resource's behavior.
- name
- The unique name of the resulting resource.
- id
- The unique provider ID of the resource to lookup.
- state
- Any extra arguments used during the lookup.
- opts
- A bag of options that control this resource's behavior.
- name
- The unique name of the resulting resource.
- id
- The unique provider ID of the resource to lookup.
- state
- Any extra arguments used during the lookup.
- opts
- A bag of options that control this resource's behavior.
- App string
- (Required)
- Create
Time string - Timestamp when the evaluation was created.
- Created
By string - The user who created the evaluation.
- Deletion
Policy string - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- Description string
- User-defined description of the evaluation.
- Display
Name string - User-defined display name of the evaluation.
- Etag string
- Etag used to ensure the object hasn't changed during a read-modify-write operation.
- Evaluation
Datasets List<string> - List of evaluation datasets the evaluation belongs to.
- Evaluation
Id string - The ID to use for the evaluation, which will become the final component of the evaluation's resource name. If not provided, a unique ID will be automatically assigned for the evaluation.
- Evaluation
Runs List<string> - The evaluation runs for this evaluation.
- Golden
Evaluation
Golden - Golden input. Structure is documented below.
- Invalid bool
- Whether the evaluation is invalid.
- Last
Updated stringBy - The user who last updated the evaluation.
- Location string
- (Required)
- Name string
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - Project string
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- Scenario
Evaluation
Scenario - Scenario input. Structure is documented below.
- List<string>
- User defined tags to categorize the evaluation.
- Update
Time string - Timestamp when the evaluation was last updated.
- App string
- (Required)
- Create
Time string - Timestamp when the evaluation was created.
- Created
By string - The user who created the evaluation.
- Deletion
Policy string - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- Description string
- User-defined description of the evaluation.
- Display
Name string - User-defined display name of the evaluation.
- Etag string
- Etag used to ensure the object hasn't changed during a read-modify-write operation.
- Evaluation
Datasets []string - List of evaluation datasets the evaluation belongs to.
- Evaluation
Id string - The ID to use for the evaluation, which will become the final component of the evaluation's resource name. If not provided, a unique ID will be automatically assigned for the evaluation.
- Evaluation
Runs []string - The evaluation runs for this evaluation.
- Golden
Evaluation
Golden Args - Golden input. Structure is documented below.
- Invalid bool
- Whether the evaluation is invalid.
- Last
Updated stringBy - The user who last updated the evaluation.
- Location string
- (Required)
- Name string
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - Project string
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- Scenario
Evaluation
Scenario Args - Scenario input. Structure is documented below.
- []string
- User defined tags to categorize the evaluation.
- Update
Time string - Timestamp when the evaluation was last updated.
- app string
- (Required)
- create_
time string - Timestamp when the evaluation was created.
- created_
by string - The user who created the evaluation.
- deletion_
policy string - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- description string
- User-defined description of the evaluation.
- display_
name string - User-defined display name of the evaluation.
- etag string
- Etag used to ensure the object hasn't changed during a read-modify-write operation.
- evaluation_
datasets list(string) - List of evaluation datasets the evaluation belongs to.
- evaluation_
id string - The ID to use for the evaluation, which will become the final component of the evaluation's resource name. If not provided, a unique ID will be automatically assigned for the evaluation.
- evaluation_
runs list(string) - The evaluation runs for this evaluation.
- golden object
- Golden input. Structure is documented below.
- invalid bool
- Whether the evaluation is invalid.
- last_
updated_ stringby - The user who last updated the evaluation.
- location string
- (Required)
- name string
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - project string
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- scenario object
- Scenario input. Structure is documented below.
- list(string)
- User defined tags to categorize the evaluation.
- update_
time string - Timestamp when the evaluation was last updated.
- app String
- (Required)
- create
Time String - Timestamp when the evaluation was created.
- created
By String - The user who created the evaluation.
- deletion
Policy String - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- description String
- User-defined description of the evaluation.
- display
Name String - User-defined display name of the evaluation.
- etag String
- Etag used to ensure the object hasn't changed during a read-modify-write operation.
- evaluation
Datasets List<String> - List of evaluation datasets the evaluation belongs to.
- evaluation
Id String - The ID to use for the evaluation, which will become the final component of the evaluation's resource name. If not provided, a unique ID will be automatically assigned for the evaluation.
- evaluation
Runs List<String> - The evaluation runs for this evaluation.
- golden
Evaluation
Golden - Golden input. Structure is documented below.
- invalid Boolean
- Whether the evaluation is invalid.
- last
Updated StringBy - The user who last updated the evaluation.
- location String
- (Required)
- name String
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - project String
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- scenario
Evaluation
Scenario - Scenario input. Structure is documented below.
- List<String>
- User defined tags to categorize the evaluation.
- update
Time String - Timestamp when the evaluation was last updated.
- app string
- (Required)
- create
Time string - Timestamp when the evaluation was created.
- created
By string - The user who created the evaluation.
- deletion
Policy string - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- description string
- User-defined description of the evaluation.
- display
Name string - User-defined display name of the evaluation.
- etag string
- Etag used to ensure the object hasn't changed during a read-modify-write operation.
- evaluation
Datasets string[] - List of evaluation datasets the evaluation belongs to.
- evaluation
Id string - The ID to use for the evaluation, which will become the final component of the evaluation's resource name. If not provided, a unique ID will be automatically assigned for the evaluation.
- evaluation
Runs string[] - The evaluation runs for this evaluation.
- golden
Evaluation
Golden - Golden input. Structure is documented below.
- invalid boolean
- Whether the evaluation is invalid.
- last
Updated stringBy - The user who last updated the evaluation.
- location string
- (Required)
- name string
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - project string
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- scenario
Evaluation
Scenario - Scenario input. Structure is documented below.
- string[]
- User defined tags to categorize the evaluation.
- update
Time string - Timestamp when the evaluation was last updated.
- app str
- (Required)
- create_
time str - Timestamp when the evaluation was created.
- created_
by str - The user who created the evaluation.
- deletion_
policy str - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- description str
- User-defined description of the evaluation.
- display_
name str - User-defined display name of the evaluation.
- etag str
- Etag used to ensure the object hasn't changed during a read-modify-write operation.
- evaluation_
datasets Sequence[str] - List of evaluation datasets the evaluation belongs to.
- evaluation_
id str - The ID to use for the evaluation, which will become the final component of the evaluation's resource name. If not provided, a unique ID will be automatically assigned for the evaluation.
- evaluation_
runs Sequence[str] - The evaluation runs for this evaluation.
- golden
Evaluation
Golden Args - Golden input. Structure is documented below.
- invalid bool
- Whether the evaluation is invalid.
- last_
updated_ strby - The user who last updated the evaluation.
- location str
- (Required)
- name str
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - project str
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- scenario
Evaluation
Scenario Args - Scenario input. Structure is documented below.
- Sequence[str]
- User defined tags to categorize the evaluation.
- update_
time str - Timestamp when the evaluation was last updated.
- app String
- (Required)
- create
Time String - Timestamp when the evaluation was created.
- created
By String - The user who created the evaluation.
- deletion
Policy String - Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. When a 'terraform destroy' or 'pulumi up' would delete the resource, the command will fail if this field is set to "PREVENT" in Terraform state. When set to "ABANDON", the command will remove the resource from Terraform management without updating or deleting the resource in the API. When set to "DELETE", deleting the resource is allowed.
- description String
- User-defined description of the evaluation.
- display
Name String - User-defined display name of the evaluation.
- etag String
- Etag used to ensure the object hasn't changed during a read-modify-write operation.
- evaluation
Datasets List<String> - List of evaluation datasets the evaluation belongs to.
- evaluation
Id String - The ID to use for the evaluation, which will become the final component of the evaluation's resource name. If not provided, a unique ID will be automatically assigned for the evaluation.
- evaluation
Runs List<String> - The evaluation runs for this evaluation.
- golden Property Map
- Golden input. Structure is documented below.
- invalid Boolean
- Whether the evaluation is invalid.
- last
Updated StringBy - The user who last updated the evaluation.
- location String
- (Required)
- name String
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - project String
- The ID of the project in which the resource belongs. If it is not provided, the provider project is used.
- scenario Property Map
- Scenario input. Structure is documented below.
- List<String>
- User defined tags to categorize the evaluation.
- update
Time String - Timestamp when the evaluation was last updated.
Supporting Types
EvaluationGolden, EvaluationGoldenArgs
- Turns
List<Evaluation
Golden Turn> - The golden turns required to replay a golden conversation. Structure is documented below.
- Evaluation
Expectations List<string> - The evaluation expectations to evaluate the replayed conversation against. Format: projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluationExpectation}
- Turns
[]Evaluation
Golden Turn - The golden turns required to replay a golden conversation. Structure is documented below.
- Evaluation
Expectations []string - The evaluation expectations to evaluate the replayed conversation against. Format: projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluationExpectation}
- turns list(object)
- The golden turns required to replay a golden conversation. Structure is documented below.
- evaluation_
expectations list(string) - The evaluation expectations to evaluate the replayed conversation against. Format: projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluationExpectation}
- turns
List<Evaluation
Golden Turn> - The golden turns required to replay a golden conversation. Structure is documented below.
- evaluation
Expectations List<String> - The evaluation expectations to evaluate the replayed conversation against. Format: projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluationExpectation}
- turns
Evaluation
Golden Turn[] - The golden turns required to replay a golden conversation. Structure is documented below.
- evaluation
Expectations string[] - The evaluation expectations to evaluate the replayed conversation against. Format: projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluationExpectation}
- turns
Sequence[Evaluation
Golden Turn] - The golden turns required to replay a golden conversation. Structure is documented below.
- evaluation_
expectations Sequence[str] - The evaluation expectations to evaluate the replayed conversation against. Format: projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluationExpectation}
- turns List<Property Map>
- The golden turns required to replay a golden conversation. Structure is documented below.
- evaluation
Expectations List<String> - The evaluation expectations to evaluate the replayed conversation against. Format: projects/{project}/locations/{location}/apps/{app}/evaluationExpectations/{evaluationExpectation}
EvaluationGoldenTurn, EvaluationGoldenTurnArgs
- Steps
List<Evaluation
Golden Turn Step> The sequence of steps required to replay a golden conversation turn. Structure is documented below.
The
rootSpanblock contains:- Root
Spans List<EvaluationGolden Turn Root Span> - (Output) The root span of the golden turn for processing and maintaining audio information. Structure is documented below.
- Steps
[]Evaluation
Golden Turn Step The sequence of steps required to replay a golden conversation turn. Structure is documented below.
The
rootSpanblock contains:- Root
Spans []EvaluationGolden Turn Root Span - (Output) The root span of the golden turn for processing and maintaining audio information. Structure is documented below.
- steps list(object)
The sequence of steps required to replay a golden conversation turn. Structure is documented below.
The
rootSpanblock contains:- root_
spans list(object) - (Output) The root span of the golden turn for processing and maintaining audio information. Structure is documented below.
- steps
List<Evaluation
Golden Turn Step> The sequence of steps required to replay a golden conversation turn. Structure is documented below.
The
rootSpanblock contains:- root
Spans List<EvaluationGolden Turn Root Span> - (Output) The root span of the golden turn for processing and maintaining audio information. Structure is documented below.
- steps
Evaluation
Golden Turn Step[] The sequence of steps required to replay a golden conversation turn. Structure is documented below.
The
rootSpanblock contains:- root
Spans EvaluationGolden Turn Root Span[] - (Output) The root span of the golden turn for processing and maintaining audio information. Structure is documented below.
- steps
Sequence[Evaluation
Golden Turn Step] The sequence of steps required to replay a golden conversation turn. Structure is documented below.
The
rootSpanblock contains:- root_
spans Sequence[EvaluationGolden Turn Root Span] - (Output) The root span of the golden turn for processing and maintaining audio information. Structure is documented below.
- steps List<Property Map>
The sequence of steps required to replay a golden conversation turn. Structure is documented below.
The
rootSpanblock contains:- root
Spans List<Property Map> - (Output) The root span of the golden turn for processing and maintaining audio information. Structure is documented below.
EvaluationGoldenTurnRootSpan, EvaluationGoldenTurnRootSpanArgs
- Attributes string
- Output only. Key-value attributes associated with the span.
- Child
Spans string - Output only. The child spans that are nested under this span.
- Duration string
- Output only. The duration of the span.
- End
Time string - Output only. The end time of the span.
- Name string
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - Start
Time string - Output only. The start time of the span.
- Attributes string
- Output only. Key-value attributes associated with the span.
- Child
Spans string - Output only. The child spans that are nested under this span.
- Duration string
- Output only. The duration of the span.
- End
Time string - Output only. The end time of the span.
- Name string
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - Start
Time string - Output only. The start time of the span.
- attributes string
- Output only. Key-value attributes associated with the span.
- child_
spans string - Output only. The child spans that are nested under this span.
- duration string
- Output only. The duration of the span.
- end_
time string - Output only. The end time of the span.
- name string
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - start_
time string - Output only. The start time of the span.
- attributes String
- Output only. Key-value attributes associated with the span.
- child
Spans String - Output only. The child spans that are nested under this span.
- duration String
- Output only. The duration of the span.
- end
Time String - Output only. The end time of the span.
- name String
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - start
Time String - Output only. The start time of the span.
- attributes string
- Output only. Key-value attributes associated with the span.
- child
Spans string - Output only. The child spans that are nested under this span.
- duration string
- Output only. The duration of the span.
- end
Time string - Output only. The end time of the span.
- name string
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - start
Time string - Output only. The start time of the span.
- attributes str
- Output only. Key-value attributes associated with the span.
- child_
spans str - Output only. The child spans that are nested under this span.
- duration str
- Output only. The duration of the span.
- end_
time str - Output only. The end time of the span.
- name str
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - start_
time str - Output only. The start time of the span.
- attributes String
- Output only. Key-value attributes associated with the span.
- child
Spans String - Output only. The child spans that are nested under this span.
- duration String
- Output only. The duration of the span.
- end
Time String - Output only. The end time of the span.
- name String
- Identifier. The unique identifier of the evaluation.
Format:
projects/{project}/locations/{location}/apps/{app}/evaluations/{evaluation} - start
Time String - Output only. The start time of the span.
EvaluationGoldenTurnStep, EvaluationGoldenTurnStepArgs
- Agent
Transfer EvaluationGolden Turn Step Agent Transfer - Represents an event indicating the transfer of the conversation to a different agent. Structure is documented below.
- Expectation
Evaluation
Golden Turn Step Expectation - Executes an expectation on the current turn to verify the agent's behavior. Note: 'user_input' and 'expectation' are mutually exclusive. Structure is documented below.
- User
Input EvaluationGolden Turn Step User Input - Represents user input for the conversation. Note: 'user_input' and 'expectation' are mutually exclusive. Structure is documented below.
- Agent
Transfer EvaluationGolden Turn Step Agent Transfer - Represents an event indicating the transfer of the conversation to a different agent. Structure is documented below.
- Expectation
Evaluation
Golden Turn Step Expectation - Executes an expectation on the current turn to verify the agent's behavior. Note: 'user_input' and 'expectation' are mutually exclusive. Structure is documented below.
- User
Input EvaluationGolden Turn Step User Input - Represents user input for the conversation. Note: 'user_input' and 'expectation' are mutually exclusive. Structure is documented below.
- agent_
transfer object - Represents an event indicating the transfer of the conversation to a different agent. Structure is documented below.
- expectation object
- Executes an expectation on the current turn to verify the agent's behavior. Note: 'user_input' and 'expectation' are mutually exclusive. Structure is documented below.
- user_
input object - Represents user input for the conversation. Note: 'user_input' and 'expectation' are mutually exclusive. Structure is documented below.
- agent
Transfer EvaluationGolden Turn Step Agent Transfer - Represents an event indicating the transfer of the conversation to a different agent. Structure is documented below.
- expectation
Evaluation
Golden Turn Step Expectation - Executes an expectation on the current turn to verify the agent's behavior. Note: 'user_input' and 'expectation' are mutually exclusive. Structure is documented below.
- user
Input EvaluationGolden Turn Step User Input - Represents user input for the conversation. Note: 'user_input' and 'expectation' are mutually exclusive. Structure is documented below.
- agent
Transfer EvaluationGolden Turn Step Agent Transfer - Represents an event indicating the transfer of the conversation to a different agent. Structure is documented below.
- expectation
Evaluation
Golden Turn Step Expectation - Executes an expectation on the current turn to verify the agent's behavior. Note: 'user_input' and 'expectation' are mutually exclusive. Structure is documented below.
- user
Input EvaluationGolden Turn Step User Input - Represents user input for the conversation. Note: 'user_input' and 'expectation' are mutually exclusive. Structure is documented below.
- agent_
transfer EvaluationGolden Turn Step Agent Transfer - Represents an event indicating the transfer of the conversation to a different agent. Structure is documented below.
- expectation
Evaluation
Golden Turn Step Expectation - Executes an expectation on the current turn to verify the agent's behavior. Note: 'user_input' and 'expectation' are mutually exclusive. Structure is documented below.
- user_
input EvaluationGolden Turn Step User Input - Represents user input for the conversation. Note: 'user_input' and 'expectation' are mutually exclusive. Structure is documented below.
- agent
Transfer Property Map - Represents an event indicating the transfer of the conversation to a different agent. Structure is documented below.
- expectation Property Map
- Executes an expectation on the current turn to verify the agent's behavior. Note: 'user_input' and 'expectation' are mutually exclusive. Structure is documented below.
- user
Input Property Map - Represents user input for the conversation. Note: 'user_input' and 'expectation' are mutually exclusive. Structure is documented below.
EvaluationGoldenTurnStepAgentTransfer, EvaluationGoldenTurnStepAgentTransferArgs
- Target
Agent string - The agent to which the conversation is being transferred.
- Display
Name string - (Output) Display name of the agent.
- Target
Agent string - The agent to which the conversation is being transferred.
- Display
Name string - (Output) Display name of the agent.
- target_
agent string - The agent to which the conversation is being transferred.
- display_
name string - (Output) Display name of the agent.
- target
Agent String - The agent to which the conversation is being transferred.
- display
Name String - (Output) Display name of the agent.
- target
Agent string - The agent to which the conversation is being transferred.
- display
Name string - (Output) Display name of the agent.
- target_
agent str - The agent to which the conversation is being transferred.
- display_
name str - (Output) Display name of the agent.
- target
Agent String - The agent to which the conversation is being transferred.
- display
Name String - (Output) Display name of the agent.
EvaluationGoldenTurnStepExpectation, EvaluationGoldenTurnStepExpectationArgs
- Agent
Response EvaluationGolden Turn Step Expectation Agent Response - Expectation on agent response. Structure is documented below.
- Agent
Transfer EvaluationGolden Turn Step Expectation Agent Transfer - Expectation on agent transfer. Structure is documented below.
- Mock
Tool EvaluationResponse Golden Turn Step Expectation Mock Tool Response - A mock response to provide if a tool is called. Structure is documented below.
- Note string
- A note describing the expectation.
- Tool
Call EvaluationGolden Turn Step Expectation Tool Call - Expectation on tool call. Structure is documented below.
- Tool
Response EvaluationGolden Turn Step Expectation Tool Response - Expectation on tool response. Structure is documented below.
- Updated
Variables EvaluationGolden Turn Step Expectation Updated Variables - Expectation on updated variables. Structure is documented below.
- Agent
Response EvaluationGolden Turn Step Expectation Agent Response - Expectation on agent response. Structure is documented below.
- Agent
Transfer EvaluationGolden Turn Step Expectation Agent Transfer - Expectation on agent transfer. Structure is documented below.
- Mock
Tool EvaluationResponse Golden Turn Step Expectation Mock Tool Response - A mock response to provide if a tool is called. Structure is documented below.
- Note string
- A note describing the expectation.
- Tool
Call EvaluationGolden Turn Step Expectation Tool Call - Expectation on tool call. Structure is documented below.
- Tool
Response EvaluationGolden Turn Step Expectation Tool Response - Expectation on tool response. Structure is documented below.
- Updated
Variables EvaluationGolden Turn Step Expectation Updated Variables - Expectation on updated variables. Structure is documented below.
- agent_
response object - Expectation on agent response. Structure is documented below.
- agent_
transfer object - Expectation on agent transfer. Structure is documented below.
- mock_
tool_ objectresponse - A mock response to provide if a tool is called. Structure is documented below.
- note string
- A note describing the expectation.
- tool_
call object - Expectation on tool call. Structure is documented below.
- tool_
response object - Expectation on tool response. Structure is documented below.
- updated_
variables object - Expectation on updated variables. Structure is documented below.
- agent
Response EvaluationGolden Turn Step Expectation Agent Response - Expectation on agent response. Structure is documented below.
- agent
Transfer EvaluationGolden Turn Step Expectation Agent Transfer - Expectation on agent transfer. Structure is documented below.
- mock
Tool EvaluationResponse Golden Turn Step Expectation Mock Tool Response - A mock response to provide if a tool is called. Structure is documented below.
- note String
- A note describing the expectation.
- tool
Call EvaluationGolden Turn Step Expectation Tool Call - Expectation on tool call. Structure is documented below.
- tool
Response EvaluationGolden Turn Step Expectation Tool Response - Expectation on tool response. Structure is documented below.
- updated
Variables EvaluationGolden Turn Step Expectation Updated Variables - Expectation on updated variables. Structure is documented below.
- agent
Response EvaluationGolden Turn Step Expectation Agent Response - Expectation on agent response. Structure is documented below.
- agent
Transfer EvaluationGolden Turn Step Expectation Agent Transfer - Expectation on agent transfer. Structure is documented below.
- mock
Tool EvaluationResponse Golden Turn Step Expectation Mock Tool Response - A mock response to provide if a tool is called. Structure is documented below.
- note string
- A note describing the expectation.
- tool
Call EvaluationGolden Turn Step Expectation Tool Call - Expectation on tool call. Structure is documented below.
- tool
Response EvaluationGolden Turn Step Expectation Tool Response - Expectation on tool response. Structure is documented below.
- updated
Variables EvaluationGolden Turn Step Expectation Updated Variables - Expectation on updated variables. Structure is documented below.
- agent_
response EvaluationGolden Turn Step Expectation Agent Response - Expectation on agent response. Structure is documented below.
- agent_
transfer EvaluationGolden Turn Step Expectation Agent Transfer - Expectation on agent transfer. Structure is documented below.
- mock_
tool_ Evaluationresponse Golden Turn Step Expectation Mock Tool Response - A mock response to provide if a tool is called. Structure is documented below.
- note str
- A note describing the expectation.
- tool_
call EvaluationGolden Turn Step Expectation Tool Call - Expectation on tool call. Structure is documented below.
- tool_
response EvaluationGolden Turn Step Expectation Tool Response - Expectation on tool response. Structure is documented below.
- updated_
variables EvaluationGolden Turn Step Expectation Updated Variables - Expectation on updated variables. Structure is documented below.
- agent
Response Property Map - Expectation on agent response. Structure is documented below.
- agent
Transfer Property Map - Expectation on agent transfer. Structure is documented below.
- mock
Tool Property MapResponse - A mock response to provide if a tool is called. Structure is documented below.
- note String
- A note describing the expectation.
- tool
Call Property Map - Expectation on tool call. Structure is documented below.
- tool
Response Property Map - Expectation on tool response. Structure is documented below.
- updated
Variables Property Map - Expectation on updated variables. Structure is documented below.
EvaluationGoldenTurnStepExpectationAgentResponse, EvaluationGoldenTurnStepExpectationAgentResponseArgs
- Chunks
List<Evaluation
Golden Turn Step Expectation Agent Response Chunk> - Content of the message as a series of chunks. Structure is documented below.
- Role string
- The role within the conversation, e.g., user, agent.
- Chunks
[]Evaluation
Golden Turn Step Expectation Agent Response Chunk - Content of the message as a series of chunks. Structure is documented below.
- Role string
- The role within the conversation, e.g., user, agent.
- chunks list(object)
- Content of the message as a series of chunks. Structure is documented below.
- role string
- The role within the conversation, e.g., user, agent.
- chunks
List<Evaluation
Golden Turn Step Expectation Agent Response Chunk> - Content of the message as a series of chunks. Structure is documented below.
- role String
- The role within the conversation, e.g., user, agent.
- chunks
Evaluation
Golden Turn Step Expectation Agent Response Chunk[] - Content of the message as a series of chunks. Structure is documented below.
- role string
- The role within the conversation, e.g., user, agent.
- chunks
Sequence[Evaluation
Golden Turn Step Expectation Agent Response Chunk] - Content of the message as a series of chunks. Structure is documented below.
- role str
- The role within the conversation, e.g., user, agent.
- chunks List<Property Map>
- Content of the message as a series of chunks. Structure is documented below.
- role String
- The role within the conversation, e.g., user, agent.
EvaluationGoldenTurnStepExpectationAgentResponseChunk, EvaluationGoldenTurnStepExpectationAgentResponseChunkArgs
- Agent
Transfer EvaluationGolden Turn Step Expectation Agent Response Chunk Agent Transfer - Represents an event indicating the transfer of a conversation to a different agent. Structure is documented below.
- Blob
Evaluation
Golden Turn Step Expectation Agent Response Chunk Blob - Represents a blob input or output in the conversation. Structure is documented below.
- Image
Evaluation
Golden Turn Step Expectation Agent Response Chunk Image - Represents an image input or output in the conversation. Structure is documented below.
- Text string
- Text data.
- Tool
Call EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Call - Request for the client or the agent to execute the specified tool. Structure is documented below.
- Tool
Response EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Response - The execution result of a specific tool from the client or the agent. Structure is documented below.
- Updated
Variables Dictionary<string, string> - Updated variables in JSON object format.
- Agent
Transfer EvaluationGolden Turn Step Expectation Agent Response Chunk Agent Transfer - Represents an event indicating the transfer of a conversation to a different agent. Structure is documented below.
- Blob
Evaluation
Golden Turn Step Expectation Agent Response Chunk Blob - Represents a blob input or output in the conversation. Structure is documented below.
- Image
Evaluation
Golden Turn Step Expectation Agent Response Chunk Image - Represents an image input or output in the conversation. Structure is documented below.
- Text string
- Text data.
- Tool
Call EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Call - Request for the client or the agent to execute the specified tool. Structure is documented below.
- Tool
Response EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Response - The execution result of a specific tool from the client or the agent. Structure is documented below.
- Updated
Variables map[string]string - Updated variables in JSON object format.
- agent_
transfer object - Represents an event indicating the transfer of a conversation to a different agent. Structure is documented below.
- blob object
- Represents a blob input or output in the conversation. Structure is documented below.
- image object
- Represents an image input or output in the conversation. Structure is documented below.
- text string
- Text data.
- tool_
call object - Request for the client or the agent to execute the specified tool. Structure is documented below.
- tool_
response object - The execution result of a specific tool from the client or the agent. Structure is documented below.
- updated_
variables map(string) - Updated variables in JSON object format.
- agent
Transfer EvaluationGolden Turn Step Expectation Agent Response Chunk Agent Transfer - Represents an event indicating the transfer of a conversation to a different agent. Structure is documented below.
- blob
Evaluation
Golden Turn Step Expectation Agent Response Chunk Blob - Represents a blob input or output in the conversation. Structure is documented below.
- image
Evaluation
Golden Turn Step Expectation Agent Response Chunk Image - Represents an image input or output in the conversation. Structure is documented below.
- text String
- Text data.
- tool
Call EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Call - Request for the client or the agent to execute the specified tool. Structure is documented below.
- tool
Response EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Response - The execution result of a specific tool from the client or the agent. Structure is documented below.
- updated
Variables Map<String,String> - Updated variables in JSON object format.
- agent
Transfer EvaluationGolden Turn Step Expectation Agent Response Chunk Agent Transfer - Represents an event indicating the transfer of a conversation to a different agent. Structure is documented below.
- blob
Evaluation
Golden Turn Step Expectation Agent Response Chunk Blob - Represents a blob input or output in the conversation. Structure is documented below.
- image
Evaluation
Golden Turn Step Expectation Agent Response Chunk Image - Represents an image input or output in the conversation. Structure is documented below.
- text string
- Text data.
- tool
Call EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Call - Request for the client or the agent to execute the specified tool. Structure is documented below.
- tool
Response EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Response - The execution result of a specific tool from the client or the agent. Structure is documented below.
- updated
Variables {[key: string]: string} - Updated variables in JSON object format.
- agent_
transfer EvaluationGolden Turn Step Expectation Agent Response Chunk Agent Transfer - Represents an event indicating the transfer of a conversation to a different agent. Structure is documented below.
- blob
Evaluation
Golden Turn Step Expectation Agent Response Chunk Blob - Represents a blob input or output in the conversation. Structure is documented below.
- image
Evaluation
Golden Turn Step Expectation Agent Response Chunk Image - Represents an image input or output in the conversation. Structure is documented below.
- text str
- Text data.
- tool_
call EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Call - Request for the client or the agent to execute the specified tool. Structure is documented below.
- tool_
response EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Response - The execution result of a specific tool from the client or the agent. Structure is documented below.
- updated_
variables Mapping[str, str] - Updated variables in JSON object format.
- agent
Transfer Property Map - Represents an event indicating the transfer of a conversation to a different agent. Structure is documented below.
- blob Property Map
- Represents a blob input or output in the conversation. Structure is documented below.
- image Property Map
- Represents an image input or output in the conversation. Structure is documented below.
- text String
- Text data.
- tool
Call Property Map - Request for the client or the agent to execute the specified tool. Structure is documented below.
- tool
Response Property Map - The execution result of a specific tool from the client or the agent. Structure is documented below.
- updated
Variables Map<String> - Updated variables in JSON object format.
EvaluationGoldenTurnStepExpectationAgentResponseChunkAgentTransfer, EvaluationGoldenTurnStepExpectationAgentResponseChunkAgentTransferArgs
- Target
Agent string - The agent to which the conversation is being transferred.
- Display
Name string - (Output) Display name of the agent.
- Target
Agent string - The agent to which the conversation is being transferred.
- Display
Name string - (Output) Display name of the agent.
- target_
agent string - The agent to which the conversation is being transferred.
- display_
name string - (Output) Display name of the agent.
- target
Agent String - The agent to which the conversation is being transferred.
- display
Name String - (Output) Display name of the agent.
- target
Agent string - The agent to which the conversation is being transferred.
- display
Name string - (Output) Display name of the agent.
- target_
agent str - The agent to which the conversation is being transferred.
- display_
name str - (Output) Display name of the agent.
- target
Agent String - The agent to which the conversation is being transferred.
- display
Name String - (Output) Display name of the agent.
EvaluationGoldenTurnStepExpectationAgentResponseChunkBlob, EvaluationGoldenTurnStepExpectationAgentResponseChunkBlobArgs
EvaluationGoldenTurnStepExpectationAgentResponseChunkImage, EvaluationGoldenTurnStepExpectationAgentResponseChunkImageArgs
EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCall, EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallArgs
- Args Dictionary<string, string>
- The input parameters and values for the tool in JSON object format.
- Display
Name string - (Output) Display name of the tool.
- Id string
- The unique identifier of the tool call.
- Tool string
- The resource name of the tool.
- Toolset
Tool EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Call Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- Args map[string]string
- The input parameters and values for the tool in JSON object format.
- Display
Name string - (Output) Display name of the tool.
- Id string
- The unique identifier of the tool call.
- Tool string
- The resource name of the tool.
- Toolset
Tool EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Call Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- args map(string)
- The input parameters and values for the tool in JSON object format.
- display_
name string - (Output) Display name of the tool.
- id string
- The unique identifier of the tool call.
- tool string
- The resource name of the tool.
- toolset_
tool object - A tool that is created from a toolset. Structure is documented below.
- args Map<String,String>
- The input parameters and values for the tool in JSON object format.
- display
Name String - (Output) Display name of the tool.
- id String
- The unique identifier of the tool call.
- tool String
- The resource name of the tool.
- toolset
Tool EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Call Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- args {[key: string]: string}
- The input parameters and values for the tool in JSON object format.
- display
Name string - (Output) Display name of the tool.
- id string
- The unique identifier of the tool call.
- tool string
- The resource name of the tool.
- toolset
Tool EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Call Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- args Mapping[str, str]
- The input parameters and values for the tool in JSON object format.
- display_
name str - (Output) Display name of the tool.
- id str
- The unique identifier of the tool call.
- tool str
- The resource name of the tool.
- toolset_
tool EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Call Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- args Map<String>
- The input parameters and values for the tool in JSON object format.
- display
Name String - (Output) Display name of the tool.
- id String
- The unique identifier of the tool call.
- tool String
- The resource name of the tool.
- toolset
Tool Property Map - A tool that is created from a toolset. Structure is documented below.
EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallToolsetTool, EvaluationGoldenTurnStepExpectationAgentResponseChunkToolCallToolsetToolArgs
EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponse, EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseArgs
- Display
Name string - (Output) Display name of the tool.
- Id string
- The matching ID of the tool call the response is for.
- Response Dictionary<string, string>
- The tool execution result in JSON object format.
- Tool string
- The resource name of the tool.
- Toolset
Tool EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Response Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- Display
Name string - (Output) Display name of the tool.
- Id string
- The matching ID of the tool call the response is for.
- Response map[string]string
- The tool execution result in JSON object format.
- Tool string
- The resource name of the tool.
- Toolset
Tool EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Response Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- display_
name string - (Output) Display name of the tool.
- id string
- The matching ID of the tool call the response is for.
- response map(string)
- The tool execution result in JSON object format.
- tool string
- The resource name of the tool.
- toolset_
tool object - A tool that is created from a toolset. Structure is documented below.
- display
Name String - (Output) Display name of the tool.
- id String
- The matching ID of the tool call the response is for.
- response Map<String,String>
- The tool execution result in JSON object format.
- tool String
- The resource name of the tool.
- toolset
Tool EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Response Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- display
Name string - (Output) Display name of the tool.
- id string
- The matching ID of the tool call the response is for.
- response {[key: string]: string}
- The tool execution result in JSON object format.
- tool string
- The resource name of the tool.
- toolset
Tool EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Response Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- display_
name str - (Output) Display name of the tool.
- id str
- The matching ID of the tool call the response is for.
- response Mapping[str, str]
- The tool execution result in JSON object format.
- tool str
- The resource name of the tool.
- toolset_
tool EvaluationGolden Turn Step Expectation Agent Response Chunk Tool Response Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- display
Name String - (Output) Display name of the tool.
- id String
- The matching ID of the tool call the response is for.
- response Map<String>
- The tool execution result in JSON object format.
- tool String
- The resource name of the tool.
- toolset
Tool Property Map - A tool that is created from a toolset. Structure is documented below.
EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseToolsetTool, EvaluationGoldenTurnStepExpectationAgentResponseChunkToolResponseToolsetToolArgs
EvaluationGoldenTurnStepExpectationAgentTransfer, EvaluationGoldenTurnStepExpectationAgentTransferArgs
- Display
Name string - (Output) Display name of the agent.
- Target
Agent string - The agent to which the conversation is being transferred.
- Display
Name string - (Output) Display name of the agent.
- Target
Agent string - The agent to which the conversation is being transferred.
- display_
name string - (Output) Display name of the agent.
- target_
agent string - The agent to which the conversation is being transferred.
- display
Name String - (Output) Display name of the agent.
- target
Agent String - The agent to which the conversation is being transferred.
- display
Name string - (Output) Display name of the agent.
- target
Agent string - The agent to which the conversation is being transferred.
- display_
name str - (Output) Display name of the agent.
- target_
agent str - The agent to which the conversation is being transferred.
- display
Name String - (Output) Display name of the agent.
- target
Agent String - The agent to which the conversation is being transferred.
EvaluationGoldenTurnStepExpectationMockToolResponse, EvaluationGoldenTurnStepExpectationMockToolResponseArgs
- Display
Name string - (Output) Output only. Display name of the tool.
- Id string
- Optional. Matching ID of the tool call.
- Response Dictionary<string, string>
- The tool execution result in JSON object format.
- Tool string
- Name of the tool to execute.
- Toolset
Tool EvaluationGolden Turn Step Expectation Mock Tool Response Toolset Tool - The toolset tool that got executed. Structure is documented below.
- Display
Name string - (Output) Output only. Display name of the tool.
- Id string
- Optional. Matching ID of the tool call.
- Response map[string]string
- The tool execution result in JSON object format.
- Tool string
- Name of the tool to execute.
- Toolset
Tool EvaluationGolden Turn Step Expectation Mock Tool Response Toolset Tool - The toolset tool that got executed. Structure is documented below.
- display_
name string - (Output) Output only. Display name of the tool.
- id string
- Optional. Matching ID of the tool call.
- response map(string)
- The tool execution result in JSON object format.
- tool string
- Name of the tool to execute.
- toolset_
tool object - The toolset tool that got executed. Structure is documented below.
- display
Name String - (Output) Output only. Display name of the tool.
- id String
- Optional. Matching ID of the tool call.
- response Map<String,String>
- The tool execution result in JSON object format.
- tool String
- Name of the tool to execute.
- toolset
Tool EvaluationGolden Turn Step Expectation Mock Tool Response Toolset Tool - The toolset tool that got executed. Structure is documented below.
- display
Name string - (Output) Output only. Display name of the tool.
- id string
- Optional. Matching ID of the tool call.
- response {[key: string]: string}
- The tool execution result in JSON object format.
- tool string
- Name of the tool to execute.
- toolset
Tool EvaluationGolden Turn Step Expectation Mock Tool Response Toolset Tool - The toolset tool that got executed. Structure is documented below.
- display_
name str - (Output) Output only. Display name of the tool.
- id str
- Optional. Matching ID of the tool call.
- response Mapping[str, str]
- The tool execution result in JSON object format.
- tool str
- Name of the tool to execute.
- toolset_
tool EvaluationGolden Turn Step Expectation Mock Tool Response Toolset Tool - The toolset tool that got executed. Structure is documented below.
- display
Name String - (Output) Output only. Display name of the tool.
- id String
- Optional. Matching ID of the tool call.
- response Map<String>
- The tool execution result in JSON object format.
- tool String
- Name of the tool to execute.
- toolset
Tool Property Map - The toolset tool that got executed. Structure is documented below.
EvaluationGoldenTurnStepExpectationMockToolResponseToolsetTool, EvaluationGoldenTurnStepExpectationMockToolResponseToolsetToolArgs
EvaluationGoldenTurnStepExpectationToolCall, EvaluationGoldenTurnStepExpectationToolCallArgs
- Args Dictionary<string, string>
- The input parameters and values for the tool in JSON object format.
- Display
Name string - (Output) Display name of the tool.
- Id string
- The unique identifier of the tool call.
- Tool string
- The resource name of the tool.
- Toolset
Tool EvaluationGolden Turn Step Expectation Tool Call Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- Args map[string]string
- The input parameters and values for the tool in JSON object format.
- Display
Name string - (Output) Display name of the tool.
- Id string
- The unique identifier of the tool call.
- Tool string
- The resource name of the tool.
- Toolset
Tool EvaluationGolden Turn Step Expectation Tool Call Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- args map(string)
- The input parameters and values for the tool in JSON object format.
- display_
name string - (Output) Display name of the tool.
- id string
- The unique identifier of the tool call.
- tool string
- The resource name of the tool.
- toolset_
tool object - A tool that is created from a toolset. Structure is documented below.
- args Map<String,String>
- The input parameters and values for the tool in JSON object format.
- display
Name String - (Output) Display name of the tool.
- id String
- The unique identifier of the tool call.
- tool String
- The resource name of the tool.
- toolset
Tool EvaluationGolden Turn Step Expectation Tool Call Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- args {[key: string]: string}
- The input parameters and values for the tool in JSON object format.
- display
Name string - (Output) Display name of the tool.
- id string
- The unique identifier of the tool call.
- tool string
- The resource name of the tool.
- toolset
Tool EvaluationGolden Turn Step Expectation Tool Call Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- args Mapping[str, str]
- The input parameters and values for the tool in JSON object format.
- display_
name str - (Output) Display name of the tool.
- id str
- The unique identifier of the tool call.
- tool str
- The resource name of the tool.
- toolset_
tool EvaluationGolden Turn Step Expectation Tool Call Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- args Map<String>
- The input parameters and values for the tool in JSON object format.
- display
Name String - (Output) Display name of the tool.
- id String
- The unique identifier of the tool call.
- tool String
- The resource name of the tool.
- toolset
Tool Property Map - A tool that is created from a toolset. Structure is documented below.
EvaluationGoldenTurnStepExpectationToolCallToolsetTool, EvaluationGoldenTurnStepExpectationToolCallToolsetToolArgs
EvaluationGoldenTurnStepExpectationToolResponse, EvaluationGoldenTurnStepExpectationToolResponseArgs
- Display
Name string - (Output) Display name of the tool.
- Id string
- The matching ID of the tool call the response is for.
- Response Dictionary<string, string>
- The tool execution result in JSON object format.
- Tool string
- The resource name of the tool.
- Toolset
Tool EvaluationGolden Turn Step Expectation Tool Response Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- Display
Name string - (Output) Display name of the tool.
- Id string
- The matching ID of the tool call the response is for.
- Response map[string]string
- The tool execution result in JSON object format.
- Tool string
- The resource name of the tool.
- Toolset
Tool EvaluationGolden Turn Step Expectation Tool Response Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- display_
name string - (Output) Display name of the tool.
- id string
- The matching ID of the tool call the response is for.
- response map(string)
- The tool execution result in JSON object format.
- tool string
- The resource name of the tool.
- toolset_
tool object - A tool that is created from a toolset. Structure is documented below.
- display
Name String - (Output) Display name of the tool.
- id String
- The matching ID of the tool call the response is for.
- response Map<String,String>
- The tool execution result in JSON object format.
- tool String
- The resource name of the tool.
- toolset
Tool EvaluationGolden Turn Step Expectation Tool Response Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- display
Name string - (Output) Display name of the tool.
- id string
- The matching ID of the tool call the response is for.
- response {[key: string]: string}
- The tool execution result in JSON object format.
- tool string
- The resource name of the tool.
- toolset
Tool EvaluationGolden Turn Step Expectation Tool Response Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- display_
name str - (Output) Display name of the tool.
- id str
- The matching ID of the tool call the response is for.
- response Mapping[str, str]
- The tool execution result in JSON object format.
- tool str
- The resource name of the tool.
- toolset_
tool EvaluationGolden Turn Step Expectation Tool Response Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- display
Name String - (Output) Display name of the tool.
- id String
- The matching ID of the tool call the response is for.
- response Map<String>
- The tool execution result in JSON object format.
- tool String
- The resource name of the tool.
- toolset
Tool Property Map - A tool that is created from a toolset. Structure is documented below.
EvaluationGoldenTurnStepExpectationToolResponseToolsetTool, EvaluationGoldenTurnStepExpectationToolResponseToolsetToolArgs
EvaluationGoldenTurnStepExpectationUpdatedVariables, EvaluationGoldenTurnStepExpectationUpdatedVariablesArgs
- Notes string
- Dummy property.
- Notes string
- Dummy property.
- notes string
- Dummy property.
- notes String
- Dummy property.
- notes string
- Dummy property.
- notes str
- Dummy property.
- notes String
- Dummy property.
EvaluationGoldenTurnStepUserInput, EvaluationGoldenTurnStepUserInputArgs
- Audio string
- Audio data from the end user.
- Blob
Evaluation
Golden Turn Step User Input Blob - Blob data from the end user. Structure is documented below.
- Dtmf string
- DTMF digits from the end user.
- Event
Evaluation
Golden Turn Step User Input Event - An event to be sent to the session. Structure is documented below.
- Image
Evaluation
Golden Turn Step User Input Image - Image data from the end user. Structure is documented below.
- Text string
- Natural language query.
- Tool
Responses EvaluationGolden Turn Step User Input Tool Responses - The list of tool execution results. Structure is documented below.
- Variables Dictionary<string, string>
- Map of variables to set.
- Will
Continue bool - Whether the session should continue.
- Audio string
- Audio data from the end user.
- Blob
Evaluation
Golden Turn Step User Input Blob - Blob data from the end user. Structure is documented below.
- Dtmf string
- DTMF digits from the end user.
- Event
Evaluation
Golden Turn Step User Input Event - An event to be sent to the session. Structure is documented below.
- Image
Evaluation
Golden Turn Step User Input Image - Image data from the end user. Structure is documented below.
- Text string
- Natural language query.
- Tool
Responses EvaluationGolden Turn Step User Input Tool Responses - The list of tool execution results. Structure is documented below.
- Variables map[string]string
- Map of variables to set.
- Will
Continue bool - Whether the session should continue.
- audio string
- Audio data from the end user.
- blob object
- Blob data from the end user. Structure is documented below.
- dtmf string
- DTMF digits from the end user.
- event object
- An event to be sent to the session. Structure is documented below.
- image object
- Image data from the end user. Structure is documented below.
- text string
- Natural language query.
- tool_
responses object - The list of tool execution results. Structure is documented below.
- variables map(string)
- Map of variables to set.
- will_
continue bool - Whether the session should continue.
- audio String
- Audio data from the end user.
- blob
Evaluation
Golden Turn Step User Input Blob - Blob data from the end user. Structure is documented below.
- dtmf String
- DTMF digits from the end user.
- event
Evaluation
Golden Turn Step User Input Event - An event to be sent to the session. Structure is documented below.
- image
Evaluation
Golden Turn Step User Input Image - Image data from the end user. Structure is documented below.
- text String
- Natural language query.
- tool
Responses EvaluationGolden Turn Step User Input Tool Responses - The list of tool execution results. Structure is documented below.
- variables Map<String,String>
- Map of variables to set.
- will
Continue Boolean - Whether the session should continue.
- audio string
- Audio data from the end user.
- blob
Evaluation
Golden Turn Step User Input Blob - Blob data from the end user. Structure is documented below.
- dtmf string
- DTMF digits from the end user.
- event
Evaluation
Golden Turn Step User Input Event - An event to be sent to the session. Structure is documented below.
- image
Evaluation
Golden Turn Step User Input Image - Image data from the end user. Structure is documented below.
- text string
- Natural language query.
- tool
Responses EvaluationGolden Turn Step User Input Tool Responses - The list of tool execution results. Structure is documented below.
- variables {[key: string]: string}
- Map of variables to set.
- will
Continue boolean - Whether the session should continue.
- audio str
- Audio data from the end user.
- blob
Evaluation
Golden Turn Step User Input Blob - Blob data from the end user. Structure is documented below.
- dtmf str
- DTMF digits from the end user.
- event
Evaluation
Golden Turn Step User Input Event - An event to be sent to the session. Structure is documented below.
- image
Evaluation
Golden Turn Step User Input Image - Image data from the end user. Structure is documented below.
- text str
- Natural language query.
- tool_
responses EvaluationGolden Turn Step User Input Tool Responses - The list of tool execution results. Structure is documented below.
- variables Mapping[str, str]
- Map of variables to set.
- will_
continue bool - Whether the session should continue.
- audio String
- Audio data from the end user.
- blob Property Map
- Blob data from the end user. Structure is documented below.
- dtmf String
- DTMF digits from the end user.
- event Property Map
- An event to be sent to the session. Structure is documented below.
- image Property Map
- Image data from the end user. Structure is documented below.
- text String
- Natural language query.
- tool
Responses Property Map - The list of tool execution results. Structure is documented below.
- variables Map<String>
- Map of variables to set.
- will
Continue Boolean - Whether the session should continue.
EvaluationGoldenTurnStepUserInputBlob, EvaluationGoldenTurnStepUserInputBlobArgs
EvaluationGoldenTurnStepUserInputEvent, EvaluationGoldenTurnStepUserInputEventArgs
- Event string
- Event name.
- Event string
- Event name.
- event string
- Event name.
- event String
- Event name.
- event string
- Event name.
- event str
- Event name.
- event String
- Event name.
EvaluationGoldenTurnStepUserInputImage, EvaluationGoldenTurnStepUserInputImageArgs
EvaluationGoldenTurnStepUserInputToolResponses, EvaluationGoldenTurnStepUserInputToolResponsesArgs
- Tool
Responses List<EvaluationGolden Turn Step User Input Tool Responses Tool Response> - The list of tool execution results. Structure is documented below.
- Tool
Responses []EvaluationGolden Turn Step User Input Tool Responses Tool Response - The list of tool execution results. Structure is documented below.
- tool_
responses list(object) - The list of tool execution results. Structure is documented below.
- tool
Responses List<EvaluationGolden Turn Step User Input Tool Responses Tool Response> - The list of tool execution results. Structure is documented below.
- tool
Responses EvaluationGolden Turn Step User Input Tool Responses Tool Response[] - The list of tool execution results. Structure is documented below.
- tool_
responses Sequence[EvaluationGolden Turn Step User Input Tool Responses Tool Response] - The list of tool execution results. Structure is documented below.
- tool
Responses List<Property Map> - The list of tool execution results. Structure is documented below.
EvaluationGoldenTurnStepUserInputToolResponsesToolResponse, EvaluationGoldenTurnStepUserInputToolResponsesToolResponseArgs
- Response Dictionary<string, string>
- Required. The tool execution result in JSON.
- Display
Name string - (Output) Output only. Display name of the tool.
- Id string
- Optional. Matching ID of the tool call.
- Tool string
- Name of the tool to execute.
- Toolset
Tool EvaluationGolden Turn Step User Input Tool Responses Tool Response Toolset Tool - The toolset tool that got executed. Structure is documented below.
- Response map[string]string
- Required. The tool execution result in JSON.
- Display
Name string - (Output) Output only. Display name of the tool.
- Id string
- Optional. Matching ID of the tool call.
- Tool string
- Name of the tool to execute.
- Toolset
Tool EvaluationGolden Turn Step User Input Tool Responses Tool Response Toolset Tool - The toolset tool that got executed. Structure is documented below.
- response map(string)
- Required. The tool execution result in JSON.
- display_
name string - (Output) Output only. Display name of the tool.
- id string
- Optional. Matching ID of the tool call.
- tool string
- Name of the tool to execute.
- toolset_
tool object - The toolset tool that got executed. Structure is documented below.
- response Map<String,String>
- Required. The tool execution result in JSON.
- display
Name String - (Output) Output only. Display name of the tool.
- id String
- Optional. Matching ID of the tool call.
- tool String
- Name of the tool to execute.
- toolset
Tool EvaluationGolden Turn Step User Input Tool Responses Tool Response Toolset Tool - The toolset tool that got executed. Structure is documented below.
- response {[key: string]: string}
- Required. The tool execution result in JSON.
- display
Name string - (Output) Output only. Display name of the tool.
- id string
- Optional. Matching ID of the tool call.
- tool string
- Name of the tool to execute.
- toolset
Tool EvaluationGolden Turn Step User Input Tool Responses Tool Response Toolset Tool - The toolset tool that got executed. Structure is documented below.
- response Mapping[str, str]
- Required. The tool execution result in JSON.
- display_
name str - (Output) Output only. Display name of the tool.
- id str
- Optional. Matching ID of the tool call.
- tool str
- Name of the tool to execute.
- toolset_
tool EvaluationGolden Turn Step User Input Tool Responses Tool Response Toolset Tool - The toolset tool that got executed. Structure is documented below.
- response Map<String>
- Required. The tool execution result in JSON.
- display
Name String - (Output) Output only. Display name of the tool.
- id String
- Optional. Matching ID of the tool call.
- tool String
- Name of the tool to execute.
- toolset
Tool Property Map - The toolset tool that got executed. Structure is documented below.
EvaluationGoldenTurnStepUserInputToolResponsesToolResponseToolsetTool, EvaluationGoldenTurnStepUserInputToolResponsesToolResponseToolsetToolArgs
EvaluationScenario, EvaluationScenarioArgs
- Rubrics List<string>
- Rubrics.
- Scenario
Expectations List<EvaluationScenario Scenario Expectation> - Scenario expectations. Structure is documented below.
- Task string
- The task to evaluate.
- Evaluation
Expectations List<string> - Evaluation expectations.
- Max
Turns int - Max turns.
- Task
Completion stringBehavior - Task completion behavior.
- User
Facts List<EvaluationScenario User Fact> - Facts about the user as a key value pair. Structure is documented below.
- User
Goal stringBehavior - User goal behavior.
- Variable
Overrides Dictionary<string, string> - Variables / Session Parameters as context for the session, keyed by variable names. Members of this struct will override any default values set by the system.
- Rubrics []string
- Rubrics.
- Scenario
Expectations []EvaluationScenario Scenario Expectation - Scenario expectations. Structure is documented below.
- Task string
- The task to evaluate.
- Evaluation
Expectations []string - Evaluation expectations.
- Max
Turns int - Max turns.
- Task
Completion stringBehavior - Task completion behavior.
- User
Facts []EvaluationScenario User Fact - Facts about the user as a key value pair. Structure is documented below.
- User
Goal stringBehavior - User goal behavior.
- Variable
Overrides map[string]string - Variables / Session Parameters as context for the session, keyed by variable names. Members of this struct will override any default values set by the system.
- rubrics list(string)
- Rubrics.
- scenario_
expectations list(object) - Scenario expectations. Structure is documented below.
- task string
- The task to evaluate.
- evaluation_
expectations list(string) - Evaluation expectations.
- max_
turns number - Max turns.
- task_
completion_ stringbehavior - Task completion behavior.
- user_
facts list(object) - Facts about the user as a key value pair. Structure is documented below.
- user_
goal_ stringbehavior - User goal behavior.
- variable_
overrides map(string) - Variables / Session Parameters as context for the session, keyed by variable names. Members of this struct will override any default values set by the system.
- rubrics List<String>
- Rubrics.
- scenario
Expectations List<EvaluationScenario Scenario Expectation> - Scenario expectations. Structure is documented below.
- task String
- The task to evaluate.
- evaluation
Expectations List<String> - Evaluation expectations.
- max
Turns Integer - Max turns.
- task
Completion StringBehavior - Task completion behavior.
- user
Facts List<EvaluationScenario User Fact> - Facts about the user as a key value pair. Structure is documented below.
- user
Goal StringBehavior - User goal behavior.
- variable
Overrides Map<String,String> - Variables / Session Parameters as context for the session, keyed by variable names. Members of this struct will override any default values set by the system.
- rubrics string[]
- Rubrics.
- scenario
Expectations EvaluationScenario Scenario Expectation[] - Scenario expectations. Structure is documented below.
- task string
- The task to evaluate.
- evaluation
Expectations string[] - Evaluation expectations.
- max
Turns number - Max turns.
- task
Completion stringBehavior - Task completion behavior.
- user
Facts EvaluationScenario User Fact[] - Facts about the user as a key value pair. Structure is documented below.
- user
Goal stringBehavior - User goal behavior.
- variable
Overrides {[key: string]: string} - Variables / Session Parameters as context for the session, keyed by variable names. Members of this struct will override any default values set by the system.
- rubrics Sequence[str]
- Rubrics.
- scenario_
expectations Sequence[EvaluationScenario Scenario Expectation] - Scenario expectations. Structure is documented below.
- task str
- The task to evaluate.
- evaluation_
expectations Sequence[str] - Evaluation expectations.
- max_
turns int - Max turns.
- task_
completion_ strbehavior - Task completion behavior.
- user_
facts Sequence[EvaluationScenario User Fact] - Facts about the user as a key value pair. Structure is documented below.
- user_
goal_ strbehavior - User goal behavior.
- variable_
overrides Mapping[str, str] - Variables / Session Parameters as context for the session, keyed by variable names. Members of this struct will override any default values set by the system.
- rubrics List<String>
- Rubrics.
- scenario
Expectations List<Property Map> - Scenario expectations. Structure is documented below.
- task String
- The task to evaluate.
- evaluation
Expectations List<String> - Evaluation expectations.
- max
Turns Number - Max turns.
- task
Completion StringBehavior - Task completion behavior.
- user
Facts List<Property Map> - Facts about the user as a key value pair. Structure is documented below.
- user
Goal StringBehavior - User goal behavior.
- variable
Overrides Map<String> - Variables / Session Parameters as context for the session, keyed by variable names. Members of this struct will override any default values set by the system.
EvaluationScenarioScenarioExpectation, EvaluationScenarioScenarioExpectationArgs
- Agent
Response EvaluationScenario Scenario Expectation Agent Response - Agent response. Structure is documented below.
- Tool
Expectation EvaluationScenario Scenario Expectation Tool Expectation - Tool expectation. Structure is documented below.
- Agent
Response EvaluationScenario Scenario Expectation Agent Response - Agent response. Structure is documented below.
- Tool
Expectation EvaluationScenario Scenario Expectation Tool Expectation - Tool expectation. Structure is documented below.
- agent_
response object - Agent response. Structure is documented below.
- tool_
expectation object - Tool expectation. Structure is documented below.
- agent
Response EvaluationScenario Scenario Expectation Agent Response - Agent response. Structure is documented below.
- tool
Expectation EvaluationScenario Scenario Expectation Tool Expectation - Tool expectation. Structure is documented below.
- agent
Response EvaluationScenario Scenario Expectation Agent Response - Agent response. Structure is documented below.
- tool
Expectation EvaluationScenario Scenario Expectation Tool Expectation - Tool expectation. Structure is documented below.
- agent_
response EvaluationScenario Scenario Expectation Agent Response - Agent response. Structure is documented below.
- tool_
expectation EvaluationScenario Scenario Expectation Tool Expectation - Tool expectation. Structure is documented below.
- agent
Response Property Map - Agent response. Structure is documented below.
- tool
Expectation Property Map - Tool expectation. Structure is documented below.
EvaluationScenarioScenarioExpectationAgentResponse, EvaluationScenarioScenarioExpectationAgentResponseArgs
- Chunks
List<Evaluation
Scenario Scenario Expectation Agent Response Chunk> - Content of the message as a series of chunks. Structure is documented below.
- Role string
- The role within the conversation, e.g., user, agent.
- Chunks
[]Evaluation
Scenario Scenario Expectation Agent Response Chunk - Content of the message as a series of chunks. Structure is documented below.
- Role string
- The role within the conversation, e.g., user, agent.
- chunks list(object)
- Content of the message as a series of chunks. Structure is documented below.
- role string
- The role within the conversation, e.g., user, agent.
- chunks
List<Evaluation
Scenario Scenario Expectation Agent Response Chunk> - Content of the message as a series of chunks. Structure is documented below.
- role String
- The role within the conversation, e.g., user, agent.
- chunks
Evaluation
Scenario Scenario Expectation Agent Response Chunk[] - Content of the message as a series of chunks. Structure is documented below.
- role string
- The role within the conversation, e.g., user, agent.
- chunks
Sequence[Evaluation
Scenario Scenario Expectation Agent Response Chunk] - Content of the message as a series of chunks. Structure is documented below.
- role str
- The role within the conversation, e.g., user, agent.
- chunks List<Property Map>
- Content of the message as a series of chunks. Structure is documented below.
- role String
- The role within the conversation, e.g., user, agent.
EvaluationScenarioScenarioExpectationAgentResponseChunk, EvaluationScenarioScenarioExpectationAgentResponseChunkArgs
- Agent
Transfer EvaluationScenario Scenario Expectation Agent Response Chunk Agent Transfer - Represents an event indicating the transfer of a conversation to a different agent. Structure is documented below.
- Blob
Evaluation
Scenario Scenario Expectation Agent Response Chunk Blob - Represents a blob input or output in the conversation. Structure is documented below.
- Image
Evaluation
Scenario Scenario Expectation Agent Response Chunk Image - Represents an image input or output in the conversation. Structure is documented below.
- Text string
- Text data.
- Tool
Call EvaluationScenario Scenario Expectation Agent Response Chunk Tool Call - Request for the client or the agent to execute the specified tool. Structure is documented below.
- Tool
Response EvaluationScenario Scenario Expectation Agent Response Chunk Tool Response - The execution result of a specific tool from the client or the agent. Structure is documented below.
- Updated
Variables Dictionary<string, string> - Updated variables in JSON object format.
- Agent
Transfer EvaluationScenario Scenario Expectation Agent Response Chunk Agent Transfer - Represents an event indicating the transfer of a conversation to a different agent. Structure is documented below.
- Blob
Evaluation
Scenario Scenario Expectation Agent Response Chunk Blob - Represents a blob input or output in the conversation. Structure is documented below.
- Image
Evaluation
Scenario Scenario Expectation Agent Response Chunk Image - Represents an image input or output in the conversation. Structure is documented below.
- Text string
- Text data.
- Tool
Call EvaluationScenario Scenario Expectation Agent Response Chunk Tool Call - Request for the client or the agent to execute the specified tool. Structure is documented below.
- Tool
Response EvaluationScenario Scenario Expectation Agent Response Chunk Tool Response - The execution result of a specific tool from the client or the agent. Structure is documented below.
- Updated
Variables map[string]string - Updated variables in JSON object format.
- agent_
transfer object - Represents an event indicating the transfer of a conversation to a different agent. Structure is documented below.
- blob object
- Represents a blob input or output in the conversation. Structure is documented below.
- image object
- Represents an image input or output in the conversation. Structure is documented below.
- text string
- Text data.
- tool_
call object - Request for the client or the agent to execute the specified tool. Structure is documented below.
- tool_
response object - The execution result of a specific tool from the client or the agent. Structure is documented below.
- updated_
variables map(string) - Updated variables in JSON object format.
- agent
Transfer EvaluationScenario Scenario Expectation Agent Response Chunk Agent Transfer - Represents an event indicating the transfer of a conversation to a different agent. Structure is documented below.
- blob
Evaluation
Scenario Scenario Expectation Agent Response Chunk Blob - Represents a blob input or output in the conversation. Structure is documented below.
- image
Evaluation
Scenario Scenario Expectation Agent Response Chunk Image - Represents an image input or output in the conversation. Structure is documented below.
- text String
- Text data.
- tool
Call EvaluationScenario Scenario Expectation Agent Response Chunk Tool Call - Request for the client or the agent to execute the specified tool. Structure is documented below.
- tool
Response EvaluationScenario Scenario Expectation Agent Response Chunk Tool Response - The execution result of a specific tool from the client or the agent. Structure is documented below.
- updated
Variables Map<String,String> - Updated variables in JSON object format.
- agent
Transfer EvaluationScenario Scenario Expectation Agent Response Chunk Agent Transfer - Represents an event indicating the transfer of a conversation to a different agent. Structure is documented below.
- blob
Evaluation
Scenario Scenario Expectation Agent Response Chunk Blob - Represents a blob input or output in the conversation. Structure is documented below.
- image
Evaluation
Scenario Scenario Expectation Agent Response Chunk Image - Represents an image input or output in the conversation. Structure is documented below.
- text string
- Text data.
- tool
Call EvaluationScenario Scenario Expectation Agent Response Chunk Tool Call - Request for the client or the agent to execute the specified tool. Structure is documented below.
- tool
Response EvaluationScenario Scenario Expectation Agent Response Chunk Tool Response - The execution result of a specific tool from the client or the agent. Structure is documented below.
- updated
Variables {[key: string]: string} - Updated variables in JSON object format.
- agent_
transfer EvaluationScenario Scenario Expectation Agent Response Chunk Agent Transfer - Represents an event indicating the transfer of a conversation to a different agent. Structure is documented below.
- blob
Evaluation
Scenario Scenario Expectation Agent Response Chunk Blob - Represents a blob input or output in the conversation. Structure is documented below.
- image
Evaluation
Scenario Scenario Expectation Agent Response Chunk Image - Represents an image input or output in the conversation. Structure is documented below.
- text str
- Text data.
- tool_
call EvaluationScenario Scenario Expectation Agent Response Chunk Tool Call - Request for the client or the agent to execute the specified tool. Structure is documented below.
- tool_
response EvaluationScenario Scenario Expectation Agent Response Chunk Tool Response - The execution result of a specific tool from the client or the agent. Structure is documented below.
- updated_
variables Mapping[str, str] - Updated variables in JSON object format.
- agent
Transfer Property Map - Represents an event indicating the transfer of a conversation to a different agent. Structure is documented below.
- blob Property Map
- Represents a blob input or output in the conversation. Structure is documented below.
- image Property Map
- Represents an image input or output in the conversation. Structure is documented below.
- text String
- Text data.
- tool
Call Property Map - Request for the client or the agent to execute the specified tool. Structure is documented below.
- tool
Response Property Map - The execution result of a specific tool from the client or the agent. Structure is documented below.
- updated
Variables Map<String> - Updated variables in JSON object format.
EvaluationScenarioScenarioExpectationAgentResponseChunkAgentTransfer, EvaluationScenarioScenarioExpectationAgentResponseChunkAgentTransferArgs
- Target
Agent string - The agent to which the conversation is being transferred.
- Display
Name string - (Output) Display name of the agent.
- Target
Agent string - The agent to which the conversation is being transferred.
- Display
Name string - (Output) Display name of the agent.
- target_
agent string - The agent to which the conversation is being transferred.
- display_
name string - (Output) Display name of the agent.
- target
Agent String - The agent to which the conversation is being transferred.
- display
Name String - (Output) Display name of the agent.
- target
Agent string - The agent to which the conversation is being transferred.
- display
Name string - (Output) Display name of the agent.
- target_
agent str - The agent to which the conversation is being transferred.
- display_
name str - (Output) Display name of the agent.
- target
Agent String - The agent to which the conversation is being transferred.
- display
Name String - (Output) Display name of the agent.
EvaluationScenarioScenarioExpectationAgentResponseChunkBlob, EvaluationScenarioScenarioExpectationAgentResponseChunkBlobArgs
EvaluationScenarioScenarioExpectationAgentResponseChunkImage, EvaluationScenarioScenarioExpectationAgentResponseChunkImageArgs
EvaluationScenarioScenarioExpectationAgentResponseChunkToolCall, EvaluationScenarioScenarioExpectationAgentResponseChunkToolCallArgs
- Args Dictionary<string, string>
- The input parameters and values for the tool in JSON object format.
- Display
Name string - (Output) Display name of the tool.
- Id string
- The unique identifier of the tool call.
- Tool string
- The resource name of the tool.
- Toolset
Tool EvaluationScenario Scenario Expectation Agent Response Chunk Tool Call Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- Args map[string]string
- The input parameters and values for the tool in JSON object format.
- Display
Name string - (Output) Display name of the tool.
- Id string
- The unique identifier of the tool call.
- Tool string
- The resource name of the tool.
- Toolset
Tool EvaluationScenario Scenario Expectation Agent Response Chunk Tool Call Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- args map(string)
- The input parameters and values for the tool in JSON object format.
- display_
name string - (Output) Display name of the tool.
- id string
- The unique identifier of the tool call.
- tool string
- The resource name of the tool.
- toolset_
tool object - A tool that is created from a toolset. Structure is documented below.
- args Map<String,String>
- The input parameters and values for the tool in JSON object format.
- display
Name String - (Output) Display name of the tool.
- id String
- The unique identifier of the tool call.
- tool String
- The resource name of the tool.
- toolset
Tool EvaluationScenario Scenario Expectation Agent Response Chunk Tool Call Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- args {[key: string]: string}
- The input parameters and values for the tool in JSON object format.
- display
Name string - (Output) Display name of the tool.
- id string
- The unique identifier of the tool call.
- tool string
- The resource name of the tool.
- toolset
Tool EvaluationScenario Scenario Expectation Agent Response Chunk Tool Call Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- args Mapping[str, str]
- The input parameters and values for the tool in JSON object format.
- display_
name str - (Output) Display name of the tool.
- id str
- The unique identifier of the tool call.
- tool str
- The resource name of the tool.
- toolset_
tool EvaluationScenario Scenario Expectation Agent Response Chunk Tool Call Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- args Map<String>
- The input parameters and values for the tool in JSON object format.
- display
Name String - (Output) Display name of the tool.
- id String
- The unique identifier of the tool call.
- tool String
- The resource name of the tool.
- toolset
Tool Property Map - A tool that is created from a toolset. Structure is documented below.
EvaluationScenarioScenarioExpectationAgentResponseChunkToolCallToolsetTool, EvaluationScenarioScenarioExpectationAgentResponseChunkToolCallToolsetToolArgs
EvaluationScenarioScenarioExpectationAgentResponseChunkToolResponse, EvaluationScenarioScenarioExpectationAgentResponseChunkToolResponseArgs
- Display
Name string - (Output) Display name of the tool.
- Id string
- The matching ID of the tool call the response is for.
- Response Dictionary<string, string>
- The tool execution result in JSON object format.
- Tool string
- The resource name of the tool.
- Toolset
Tool EvaluationScenario Scenario Expectation Agent Response Chunk Tool Response Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- Display
Name string - (Output) Display name of the tool.
- Id string
- The matching ID of the tool call the response is for.
- Response map[string]string
- The tool execution result in JSON object format.
- Tool string
- The resource name of the tool.
- Toolset
Tool EvaluationScenario Scenario Expectation Agent Response Chunk Tool Response Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- display_
name string - (Output) Display name of the tool.
- id string
- The matching ID of the tool call the response is for.
- response map(string)
- The tool execution result in JSON object format.
- tool string
- The resource name of the tool.
- toolset_
tool object - A tool that is created from a toolset. Structure is documented below.
- display
Name String - (Output) Display name of the tool.
- id String
- The matching ID of the tool call the response is for.
- response Map<String,String>
- The tool execution result in JSON object format.
- tool String
- The resource name of the tool.
- toolset
Tool EvaluationScenario Scenario Expectation Agent Response Chunk Tool Response Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- display
Name string - (Output) Display name of the tool.
- id string
- The matching ID of the tool call the response is for.
- response {[key: string]: string}
- The tool execution result in JSON object format.
- tool string
- The resource name of the tool.
- toolset
Tool EvaluationScenario Scenario Expectation Agent Response Chunk Tool Response Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- display_
name str - (Output) Display name of the tool.
- id str
- The matching ID of the tool call the response is for.
- response Mapping[str, str]
- The tool execution result in JSON object format.
- tool str
- The resource name of the tool.
- toolset_
tool EvaluationScenario Scenario Expectation Agent Response Chunk Tool Response Toolset Tool - A tool that is created from a toolset. Structure is documented below.
- display
Name String - (Output) Display name of the tool.
- id String
- The matching ID of the tool call the response is for.
- response Map<String>
- The tool execution result in JSON object format.
- tool String
- The resource name of the tool.
- toolset
Tool Property Map - A tool that is created from a toolset. Structure is documented below.
EvaluationScenarioScenarioExpectationAgentResponseChunkToolResponseToolsetTool, EvaluationScenarioScenarioExpectationAgentResponseChunkToolResponseToolsetToolArgs
EvaluationScenarioScenarioExpectationToolExpectation, EvaluationScenarioScenarioExpectationToolExpectationArgs
- Expected
Tool EvaluationCall Scenario Scenario Expectation Tool Expectation Expected Tool Call - Expected tool call. Structure is documented below.
- Mock
Tool EvaluationResponse Scenario Scenario Expectation Tool Expectation Mock Tool Response - Mock tool response. Structure is documented below.
- Expected
Tool EvaluationCall Scenario Scenario Expectation Tool Expectation Expected Tool Call - Expected tool call. Structure is documented below.
- Mock
Tool EvaluationResponse Scenario Scenario Expectation Tool Expectation Mock Tool Response - Mock tool response. Structure is documented below.
- expected_
tool_ objectcall - Expected tool call. Structure is documented below.
- mock_
tool_ objectresponse - Mock tool response. Structure is documented below.
- expected
Tool EvaluationCall Scenario Scenario Expectation Tool Expectation Expected Tool Call - Expected tool call. Structure is documented below.
- mock
Tool EvaluationResponse Scenario Scenario Expectation Tool Expectation Mock Tool Response - Mock tool response. Structure is documented below.
- expected
Tool EvaluationCall Scenario Scenario Expectation Tool Expectation Expected Tool Call - Expected tool call. Structure is documented below.
- mock
Tool EvaluationResponse Scenario Scenario Expectation Tool Expectation Mock Tool Response - Mock tool response. Structure is documented below.
- expected_
tool_ Evaluationcall Scenario Scenario Expectation Tool Expectation Expected Tool Call - Expected tool call. Structure is documented below.
- mock_
tool_ Evaluationresponse Scenario Scenario Expectation Tool Expectation Mock Tool Response - Mock tool response. Structure is documented below.
- expected
Tool Property MapCall - Expected tool call. Structure is documented below.
- mock
Tool Property MapResponse - Mock tool response. Structure is documented below.
EvaluationScenarioScenarioExpectationToolExpectationExpectedToolCall, EvaluationScenarioScenarioExpectationToolExpectationExpectedToolCallArgs
- Args Dictionary<string, string>
- The input parameters and values for the tool in JSON object format.
- Display
Name string - (Output) Output only. Display name of the tool.
- Id string
- Optional. The unique identifier of the tool call.
- Tool string
- Name of the tool.
- Toolset
Tool EvaluationScenario Scenario Expectation Tool Expectation Expected Tool Call Toolset Tool - The toolset tool. Structure is documented below.
- Args map[string]string
- The input parameters and values for the tool in JSON object format.
- Display
Name string - (Output) Output only. Display name of the tool.
- Id string
- Optional. The unique identifier of the tool call.
- Tool string
- Name of the tool.
- Toolset
Tool EvaluationScenario Scenario Expectation Tool Expectation Expected Tool Call Toolset Tool - The toolset tool. Structure is documented below.
- args map(string)
- The input parameters and values for the tool in JSON object format.
- display_
name string - (Output) Output only. Display name of the tool.
- id string
- Optional. The unique identifier of the tool call.
- tool string
- Name of the tool.
- toolset_
tool object - The toolset tool. Structure is documented below.
- args Map<String,String>
- The input parameters and values for the tool in JSON object format.
- display
Name String - (Output) Output only. Display name of the tool.
- id String
- Optional. The unique identifier of the tool call.
- tool String
- Name of the tool.
- toolset
Tool EvaluationScenario Scenario Expectation Tool Expectation Expected Tool Call Toolset Tool - The toolset tool. Structure is documented below.
- args {[key: string]: string}
- The input parameters and values for the tool in JSON object format.
- display
Name string - (Output) Output only. Display name of the tool.
- id string
- Optional. The unique identifier of the tool call.
- tool string
- Name of the tool.
- toolset
Tool EvaluationScenario Scenario Expectation Tool Expectation Expected Tool Call Toolset Tool - The toolset tool. Structure is documented below.
- args Mapping[str, str]
- The input parameters and values for the tool in JSON object format.
- display_
name str - (Output) Output only. Display name of the tool.
- id str
- Optional. The unique identifier of the tool call.
- tool str
- Name of the tool.
- toolset_
tool EvaluationScenario Scenario Expectation Tool Expectation Expected Tool Call Toolset Tool - The toolset tool. Structure is documented below.
- args Map<String>
- The input parameters and values for the tool in JSON object format.
- display
Name String - (Output) Output only. Display name of the tool.
- id String
- Optional. The unique identifier of the tool call.
- tool String
- Name of the tool.
- toolset
Tool Property Map - The toolset tool. Structure is documented below.
EvaluationScenarioScenarioExpectationToolExpectationExpectedToolCallToolsetTool, EvaluationScenarioScenarioExpectationToolExpectationExpectedToolCallToolsetToolArgs
EvaluationScenarioScenarioExpectationToolExpectationMockToolResponse, EvaluationScenarioScenarioExpectationToolExpectationMockToolResponseArgs
- Display
Name string - (Output) Output only. Display name of the tool.
- Id string
- Optional. Matching ID of the tool call.
- Response Dictionary<string, string>
- The tool execution result in JSON object format.
- Tool string
- Name of the tool to execute.
- Toolset
Tool EvaluationScenario Scenario Expectation Tool Expectation Mock Tool Response Toolset Tool - The toolset tool that got executed. Structure is documented below.
- Display
Name string - (Output) Output only. Display name of the tool.
- Id string
- Optional. Matching ID of the tool call.
- Response map[string]string
- The tool execution result in JSON object format.
- Tool string
- Name of the tool to execute.
- Toolset
Tool EvaluationScenario Scenario Expectation Tool Expectation Mock Tool Response Toolset Tool - The toolset tool that got executed. Structure is documented below.
- display_
name string - (Output) Output only. Display name of the tool.
- id string
- Optional. Matching ID of the tool call.
- response map(string)
- The tool execution result in JSON object format.
- tool string
- Name of the tool to execute.
- toolset_
tool object - The toolset tool that got executed. Structure is documented below.
- display
Name String - (Output) Output only. Display name of the tool.
- id String
- Optional. Matching ID of the tool call.
- response Map<String,String>
- The tool execution result in JSON object format.
- tool String
- Name of the tool to execute.
- toolset
Tool EvaluationScenario Scenario Expectation Tool Expectation Mock Tool Response Toolset Tool - The toolset tool that got executed. Structure is documented below.
- display
Name string - (Output) Output only. Display name of the tool.
- id string
- Optional. Matching ID of the tool call.
- response {[key: string]: string}
- The tool execution result in JSON object format.
- tool string
- Name of the tool to execute.
- toolset
Tool EvaluationScenario Scenario Expectation Tool Expectation Mock Tool Response Toolset Tool - The toolset tool that got executed. Structure is documented below.
- display_
name str - (Output) Output only. Display name of the tool.
- id str
- Optional. Matching ID of the tool call.
- response Mapping[str, str]
- The tool execution result in JSON object format.
- tool str
- Name of the tool to execute.
- toolset_
tool EvaluationScenario Scenario Expectation Tool Expectation Mock Tool Response Toolset Tool - The toolset tool that got executed. Structure is documented below.
- display
Name String - (Output) Output only. Display name of the tool.
- id String
- Optional. Matching ID of the tool call.
- response Map<String>
- The tool execution result in JSON object format.
- tool String
- Name of the tool to execute.
- toolset
Tool Property Map - The toolset tool that got executed. Structure is documented below.
EvaluationScenarioScenarioExpectationToolExpectationMockToolResponseToolsetTool, EvaluationScenarioScenarioExpectationToolExpectationMockToolResponseToolsetToolArgs
EvaluationScenarioUserFact, EvaluationScenarioUserFactArgs
Import
Evaluation can be imported using any of these accepted formats:
projects/{{project}}/locations/{{location}}/apps/{{app}}/evaluations/{{evaluation_id}}{{project}}/{{location}}/{{app}}/{{evaluation_id}}{{location}}/{{app}}/{{evaluation_id}}
When using the pulumi import command, Evaluation can be imported using one of the formats above. For example:
$ pulumi import gcp:ces/evaluation:Evaluation default projects/{{project}}/locations/{{location}}/apps/{{app}}/evaluations/{{evaluation_id}}
$ pulumi import gcp:ces/evaluation:Evaluation default {{project}}/{{location}}/{{app}}/{{evaluation_id}}
$ pulumi import gcp:ces/evaluation:Evaluation default {{location}}/{{app}}/{{evaluation_id}}
To learn more about importing existing cloud resources, see Importing resources.
Package Details
- Repository
- Google Cloud (GCP) Classic pulumi/pulumi-gcp
- License
- Apache-2.0
- Notes
- This Pulumi package is based on the
google-betaTerraform Provider.
Viewing docs for Google Cloud v9.24.0
published on Friday, May 22, 2026 by Pulumi
published on Friday, May 22, 2026 by Pulumi