diff --git a/cli/azd/extensions/azure.ai.agents/README.md b/cli/azd/extensions/azure.ai.agents/README.md index 80df72db3ac..ea196777601 100644 --- a/cli/azd/extensions/azure.ai.agents/README.md +++ b/cli/azd/extensions/azure.ai.agents/README.md @@ -133,6 +133,104 @@ Details: > the other inline agent properties such as `codeConfiguration` and > `environmentVariables`. +## Prompt voice agent configuration + +Prompt voice agents keep their editable definition on the `azure.ai.agent` +service entry in `azure.yaml`. A minimal managed model agent only needs `kind`, +`model`, and `name`; advanced voice settings can be layered on the same service +without changing hosted-agent projects. The shape below follows the prompt voice +service contract used by the current samples and the Vienna implementation; azd +keeps common fields strongly typed and passes extensible tool/avatar details to +the service for final validation. + +```yaml +services: + voice-agent: + host: azure.ai.agent + project: src/voice-agent + kind: prompt-voice + name: voice-agent + modelType: managed # or self_deployed for BYOM + model: + id: gpt-realtime + instructions: You are {{agent_persona}}, a concise support agent. + structuredInputs: + agent_persona: + type: string + defaultValue: Ada + audio: + input: + format: + type: audio/pcm + rate: 24000 + noiseReduction: + type: near_field + turnDetection: + type: server_vad + threshold: 0.5 + prefixPaddingMs: 300 + silenceDurationMs: 500 + createResponse: true + transcription: + model: whisper-1 + language: en-US + prompt: Contoso product names + output: + format: + type: audio/pcm + rate: 24000 + voice: + type: azure_standard + name: en-US-AvaNeural + style: cheerful + speed: 1.0 + outputModalities: [audio] + store: true + tools: + - type: system + name: end_conversation + - type: function + name: get_weather + description: Get weather for a city + parameters: + type: object + properties: {} + avatar: + type: video-avatar + character: lisa + style: casual-sitting + output_protocol: webrtc +``` + +Details: + +- Existing simple fields continue to work: `instructions`, `voice`, and `store` + are shorthand for the common settings. If `audio.output.voice` is present, it + takes precedence over the shorthand `voice` field. +- Missing `audio` fields keep azd defaults: PCM audio at 24 kHz, server VAD, + `azure-speech` transcription, and the default Azure Neural voice. +- Supported `audio.format.type` values are `audio/pcm`, `audio/pcmu`, and + `audio/pcma`; `audio.output.speed` must be between `0.25` and `1.5`. +- `turnDetection.type` supports `server_vad` with `threshold`, + `prefixPaddingMs`, `silenceDurationMs`, and `createResponse`, or + `semantic_vad` with `eagerness` (`auto`, `low`, `medium`, or `high`). +- `outputModalities` well-known values are `audio`, `text`, `animation`, and + `avatar`. The service defaults to audio when omitted. +- `structuredInputs` follows the prompt agent structured input shape: + `description`, `defaultValue`, `schema`, and `required`. azd converts + `defaultValue` to the service wire field `default_value` when deploying. +- Direct voice tool types are `function`, `mcp`, `system`, and `toolbox`. + Server-side tools such as `web_search`, `azure_ai_search`, and `openapi` must + be packaged in a toolbox instead of listed directly on the voice agent. +- `avatar` well-known fields are `type`, `character`, `style`, `customized`, and + `output_protocol`; `type` and `character` are required by the service when an + avatar is configured. Well-known avatar protocols are `webrtc` and `websocket`. +- `tools` and `avatar` intentionally remain light pass-through blocks so new + service-side capabilities can be adopted without adding azd command flags. +- In deprecated standalone `agent.yaml` files the equivalent keys use snake_case, + for example `model_type`, `structured_inputs`, `output_modalities`, + `turn_detection`, and `prefix_padding_ms`. + ## Session idle timeout A hosted agent's runtime session sandbox is suspended by Foundry after a period diff --git a/cli/azd/extensions/azure.ai.agents/cspell.yaml b/cli/azd/extensions/azure.ai.agents/cspell.yaml index 5d2d9e0950c..78be91d2fea 100644 --- a/cli/azd/extensions/azure.ai.agents/cspell.yaml +++ b/cli/azd/extensions/azure.ai.agents/cspell.yaml @@ -11,6 +11,9 @@ words: # Voice (prompt-voice) agents - BYOM - Nanami + - pcma + - pcmu + - webrtc # Azure region names - australiaeast - brazilsouth diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go index a276ef6559c..885b09c20c1 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go @@ -312,7 +312,7 @@ const ( // VoiceAudioFormat describes a PCM audio stream format (e.g. audio/pcm @ 24 kHz). type VoiceAudioFormat struct { Type string `json:"type"` - Rate int `json:"rate"` + Rate *int `json:"rate,omitempty"` } // VoiceTurnDetection configures server-side voice-activity detection so the @@ -322,32 +322,47 @@ type VoiceTurnDetection struct { Threshold *float64 `json:"threshold,omitempty"` PrefixPaddingMs *int `json:"prefix_padding_ms,omitempty"` SilenceDurationMs *int `json:"silence_duration_ms,omitempty"` + CreateResponse *bool `json:"create_response,omitempty"` + Eagerness *string `json:"eagerness,omitempty"` } // VoiceTranscription enables user-speech transcription events on the input stream. type VoiceTranscription struct { - Model string `json:"model,omitempty"` + Model string `json:"model,omitempty"` + Language *string `json:"language,omitempty"` + Prompt *string `json:"prompt,omitempty"` +} + +// VoiceNoiseReduction configures input audio noise reduction. +type VoiceNoiseReduction struct { + Type string `json:"type"` } // VoiceInputConfig is the input (caller -> agent) audio configuration. type VoiceInputConfig struct { - Format *VoiceAudioFormat `json:"format,omitempty"` - TurnDetection *VoiceTurnDetection `json:"turn_detection,omitempty"` - Transcription *VoiceTranscription `json:"transcription,omitempty"` + Format *VoiceAudioFormat `json:"format,omitempty"` + NoiseReduction *VoiceNoiseReduction `json:"noise_reduction,omitempty"` + TurnDetection *VoiceTurnDetection `json:"turn_detection,omitempty"` + Transcription *VoiceTranscription `json:"transcription,omitempty"` } // VoiceConfig selects the output voice. Type is "openai" for realtime voices // (single lowercase word, e.g. "alloy") or "azure_standard" for Azure Neural // voices (e.g. "en-US-Ava:DragonHDLatestNeural"). type VoiceConfig struct { - Type string `json:"type"` - Name string `json:"name"` + Type string `json:"type"` + Name string `json:"name"` + Style *string `json:"style,omitempty"` + Pitch *string `json:"pitch,omitempty"` + Rate *string `json:"rate,omitempty"` + Locale *string `json:"locale,omitempty"` } // VoiceOutputConfig is the output (agent -> caller) audio configuration. type VoiceOutputConfig struct { Format *VoiceAudioFormat `json:"format,omitempty"` Voice *VoiceConfig `json:"voice,omitempty"` + Speed *float64 `json:"speed,omitempty"` } // VoiceAudioConfig bundles the input and output audio configuration. @@ -364,9 +379,12 @@ type VoiceAgentDefinition struct { ModelType VoiceModelType `json:"model_type"` Model string `json:"model"` Instructions string `json:"instructions,omitempty"` + StructuredInputs map[string]any `json:"structured_inputs,omitempty"` Audio *VoiceAudioConfig `json:"audio,omitempty"` OutputModalities []string `json:"output_modalities,omitempty"` Store *bool `json:"store,omitempty"` + Tools []map[string]any `json:"tools,omitempty"` + Avatar map[string]any `json:"avatar,omitempty"` } // CreateAgentVersionRequest represents a request to create an agent version diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go index 8a385d27632..d84867a92dd 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go @@ -553,6 +553,102 @@ func buildVoiceConfig(name string) *agent_api.VoiceConfig { return &agent_api.VoiceConfig{Type: "azure_standard", Name: trimmed} } +func defaultVoiceAudioFormat() *agent_api.VoiceAudioFormat { + rate := defaultVoiceAudioRate + return &agent_api.VoiceAudioFormat{Type: defaultVoiceAudioType, Rate: &rate} +} + +func mapVoiceAudioFormat(format *VoiceAudioFormat, fallback *agent_api.VoiceAudioFormat) *agent_api.VoiceAudioFormat { + out := &agent_api.VoiceAudioFormat{} + if fallback != nil { + *out = *fallback + } + if format != nil { + if strings.TrimSpace(format.Type) != "" { + out.Type = strings.TrimSpace(format.Type) + } + if format.Rate != nil { + out.Rate = format.Rate + } + } + return out +} + +func mapVoiceTurnDetection(turnDetection *VoiceTurnDetection) *agent_api.VoiceTurnDetection { + out := &agent_api.VoiceTurnDetection{Type: defaultVoiceTurnDetectionType} + if turnDetection == nil { + return out + } + if strings.TrimSpace(turnDetection.Type) != "" { + out.Type = strings.TrimSpace(turnDetection.Type) + } + out.Threshold = turnDetection.Threshold + out.PrefixPaddingMs = turnDetection.PrefixPaddingMs + out.SilenceDurationMs = turnDetection.SilenceDurationMs + out.CreateResponse = turnDetection.CreateResponse + out.Eagerness = turnDetection.Eagerness + return out +} + +func mapVoiceTranscription(transcription *VoiceTranscription) *agent_api.VoiceTranscription { + out := &agent_api.VoiceTranscription{Model: defaultVoiceInputTranscriptionModel} + if transcription == nil { + return out + } + if strings.TrimSpace(transcription.Model) != "" { + out.Model = strings.TrimSpace(transcription.Model) + } + out.Language = transcription.Language + out.Prompt = transcription.Prompt + return out +} + +func mapVoiceConfig(voice *VoiceConfig, fallbackName string) *agent_api.VoiceConfig { + if voice == nil { + return buildVoiceConfig(fallbackName) + } + name := strings.TrimSpace(voice.Name) + if name == "" { + name = fallbackName + } + voiceType := strings.TrimSpace(voice.Type) + if voiceType == "" { + return buildVoiceConfig(name) + } + return &agent_api.VoiceConfig{ + Type: voiceType, + Name: name, + Style: voice.Style, + Pitch: voice.Pitch, + Rate: voice.Rate, + Locale: voice.Locale, + } +} + +func mapVoiceStructuredInputs(inputs map[string]any) map[string]any { + if len(inputs) == 0 { + return nil + } + out := make(map[string]any, len(inputs)) + for name, input := range inputs { + inputMap, ok := input.(map[string]any) + if !ok { + out[name] = input + continue + } + + mapped := maps.Clone(inputMap) + if value, ok := mapped["defaultValue"]; ok { + if _, hasSnakeCase := mapped["default_value"]; !hasSnakeCase { + mapped["default_value"] = value + } + delete(mapped, "defaultValue") + } + out[name] = mapped + } + return out +} + // CreateVoiceAgentAPIRequest builds a CreateAgentRequest for a declarative // voice agent. It translates the authoring kind "prompt-voice" into the // data-plane service kind "voice" and defaults the audio pipeline. @@ -574,6 +670,9 @@ func CreateVoiceAgentAPIRequest(voiceAgent VoiceAgent) (*agent_api.CreateAgentRe "model_type '%s' is not supported; use '%s' or '%s'", voiceAgent.ModelType, VoiceModelTypeManaged, VoiceModelTypeSelfDeployed) } + if errors := validateVoiceAgentAdvancedConfig(voiceAgent); len(errors) > 0 { + return nil, fmt.Errorf("invalid prompt-voice configuration: %s", strings.Join(errors, "; ")) + } instructions := defaultVoiceInstructions if voiceAgent.Instructions != nil && *voiceAgent.Instructions != "" { @@ -585,9 +684,32 @@ func CreateVoiceAgentAPIRequest(voiceAgent VoiceAgent) (*agent_api.CreateAgentRe voiceName = *voiceAgent.Voice } - audioFormat := &agent_api.VoiceAudioFormat{ - Type: defaultVoiceAudioType, - Rate: defaultVoiceAudioRate, + inputFormat := defaultVoiceAudioFormat() + outputFormat := defaultVoiceAudioFormat() + turnDetection := mapVoiceTurnDetection(nil) + transcription := mapVoiceTranscription(nil) + var noiseReduction *agent_api.VoiceNoiseReduction + outputVoice := buildVoiceConfig(voiceName) + var outputSpeed *float64 + if voiceAgent.Audio != nil { + if voiceAgent.Audio.Input != nil { + inputFormat = mapVoiceAudioFormat(voiceAgent.Audio.Input.Format, inputFormat) + if voiceAgent.Audio.Input.NoiseReduction != nil { + noiseReduction = &agent_api.VoiceNoiseReduction{Type: strings.TrimSpace(voiceAgent.Audio.Input.NoiseReduction.Type)} + } + turnDetection = mapVoiceTurnDetection(voiceAgent.Audio.Input.TurnDetection) + transcription = mapVoiceTranscription(voiceAgent.Audio.Input.Transcription) + } + if voiceAgent.Audio.Output != nil { + outputFormat = mapVoiceAudioFormat(voiceAgent.Audio.Output.Format, outputFormat) + outputVoice = mapVoiceConfig(voiceAgent.Audio.Output.Voice, voiceName) + outputSpeed = voiceAgent.Audio.Output.Speed + } + } + + outputModalities := []string{"audio"} + if len(voiceAgent.OutputModalities) > 0 { + outputModalities = voiceAgent.OutputModalities } voiceDef := agent_api.VoiceAgentDefinition{ @@ -595,22 +717,27 @@ func CreateVoiceAgentAPIRequest(voiceAgent VoiceAgent) (*agent_api.CreateAgentRe // Translate authoring kind prompt-voice -> service kind voice. Kind: agent_api.AgentKindVoice, }, - ModelType: modelType, - Model: modelID, - Instructions: instructions, + ModelType: modelType, + Model: modelID, + Instructions: instructions, + StructuredInputs: mapVoiceStructuredInputs(voiceAgent.StructuredInputs), Audio: &agent_api.VoiceAudioConfig{ Input: &agent_api.VoiceInputConfig{ - Format: audioFormat, - TurnDetection: &agent_api.VoiceTurnDetection{Type: defaultVoiceTurnDetectionType}, - Transcription: &agent_api.VoiceTranscription{Model: defaultVoiceInputTranscriptionModel}, + Format: inputFormat, + NoiseReduction: noiseReduction, + TurnDetection: turnDetection, + Transcription: transcription, }, Output: &agent_api.VoiceOutputConfig{ - Format: audioFormat, - Voice: buildVoiceConfig(voiceName), + Format: outputFormat, + Voice: outputVoice, + Speed: outputSpeed, }, }, - OutputModalities: []string{"audio"}, + OutputModalities: outputModalities, Store: voiceAgent.Store, + Tools: voiceAgent.Tools, + Avatar: voiceAgent.Avatar, } return createAgentAPIRequest(voiceAgent.AgentDefinition, voiceDef, nil, nil) diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go index 137e9b6e847..d949b28473a 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go @@ -117,7 +117,8 @@ func TestCreateVoiceAgentAPIRequest_Defaults(t *testing.T) { t.Fatalf("Audio pipeline not populated: %+v", def.Audio) } in := def.Audio.Input - if in.Format == nil || in.Format.Type != defaultVoiceAudioType || in.Format.Rate != defaultVoiceAudioRate { + if in.Format == nil || in.Format.Type != defaultVoiceAudioType || in.Format.Rate == nil || + *in.Format.Rate != defaultVoiceAudioRate { t.Errorf("input format = %+v", in.Format) } if in.TurnDetection == nil || in.TurnDetection.Type != defaultVoiceTurnDetectionType { @@ -127,7 +128,8 @@ func TestCreateVoiceAgentAPIRequest_Defaults(t *testing.T) { t.Errorf("transcription = %+v", in.Transcription) } out := def.Audio.Output - if out.Format == nil || out.Format.Type != defaultVoiceAudioType || out.Format.Rate != defaultVoiceAudioRate { + if out.Format == nil || out.Format.Type != defaultVoiceAudioType || out.Format.Rate == nil || + *out.Format.Rate != defaultVoiceAudioRate { t.Errorf("output format = %+v", out.Format) } // Default voice is the DragonHD Azure Neural voice. @@ -172,6 +174,100 @@ func TestCreateVoiceAgentAPIRequest_Overrides(t *testing.T) { } } +func TestCreateVoiceAgentAPIRequest_AdvancedConfig(t *testing.T) { + t.Parallel() + inRate := 16000 + outRate := 24000 + threshold := 0.4 + prefixPaddingMs := 250 + silenceDurationMs := 600 + createResponse := false + language := "en-US" + prompt := "Contoso product names" + speed := 1.2 + store := true + voiceStyle := "cheerful" + agent := VoiceAgent{ + AgentDefinition: AgentDefinition{Kind: AgentKindPromptVoice, Name: "advanced-voice"}, + Model: &Model{Id: "gpt-realtime"}, + StructuredInputs: map[string]any{ + "agent_persona": map[string]any{"type": "string", "defaultValue": "Ada"}, + }, + Audio: &VoiceAudio{ + Input: &VoiceAudioInput{ + Format: &VoiceAudioFormat{Type: "audio/pcmu", Rate: &inRate}, + NoiseReduction: &VoiceNoiseReduction{Type: "near_field"}, + TurnDetection: &VoiceTurnDetection{ + Type: "server_vad", + Threshold: &threshold, + PrefixPaddingMs: &prefixPaddingMs, + SilenceDurationMs: &silenceDurationMs, + CreateResponse: &createResponse, + }, + Transcription: &VoiceTranscription{Model: "whisper-1", Language: &language, Prompt: &prompt}, + }, + Output: &VoiceAudioOutput{ + Format: &VoiceAudioFormat{Type: "audio/pcm", Rate: &outRate}, + Voice: &VoiceConfig{Type: "azure_standard", Name: "en-US-AvaNeural", Style: &voiceStyle}, + Speed: &speed, + }, + }, + OutputModalities: []string{"audio", "text"}, + Store: &store, + Tools: []map[string]any{{"type": "system", "name": "end_conversation"}}, + Avatar: map[string]any{"type": "video-avatar", "character": "lisa"}, + } + + req, err := CreateVoiceAgentAPIRequest(agent) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + def := req.Definition.(agent_api.VoiceAgentDefinition) + persona, ok := def.StructuredInputs["agent_persona"].(map[string]any) + if !ok { + t.Fatalf("structured inputs not mapped: %+v", def.StructuredInputs) + } + if persona["default_value"] != "Ada" || persona["defaultValue"] != nil { + t.Fatalf("structured input default not converted to service shape: %+v", persona) + } + if def.Audio.Input.Format.Type != "audio/pcmu" || *def.Audio.Input.Format.Rate != inRate { + t.Errorf("input format = %+v", def.Audio.Input.Format) + } + if def.Audio.Input.NoiseReduction == nil || def.Audio.Input.NoiseReduction.Type != "near_field" { + t.Errorf("noise reduction = %+v", def.Audio.Input.NoiseReduction) + } + if def.Audio.Input.TurnDetection.Threshold != &threshold || + def.Audio.Input.TurnDetection.PrefixPaddingMs != &prefixPaddingMs || + def.Audio.Input.TurnDetection.SilenceDurationMs != &silenceDurationMs || + def.Audio.Input.TurnDetection.CreateResponse != &createResponse { + t.Errorf("turn detection = %+v", def.Audio.Input.TurnDetection) + } + if def.Audio.Input.Transcription.Model != "whisper-1" || + def.Audio.Input.Transcription.Language != &language || + def.Audio.Input.Transcription.Prompt != &prompt { + t.Errorf("transcription = %+v", def.Audio.Input.Transcription) + } + if def.Audio.Output.Format.Type != "audio/pcm" || *def.Audio.Output.Format.Rate != outRate { + t.Errorf("output format = %+v", def.Audio.Output.Format) + } + if def.Audio.Output.Voice.Type != "azure_standard" || def.Audio.Output.Voice.Name != "en-US-AvaNeural" || + def.Audio.Output.Voice.Style != &voiceStyle { + t.Errorf("voice = %+v", def.Audio.Output.Voice) + } + if def.Audio.Output.Speed != &speed { + t.Errorf("speed = %v", def.Audio.Output.Speed) + } + if len(def.OutputModalities) != 2 || def.OutputModalities[1] != "text" { + t.Errorf("output modalities = %v", def.OutputModalities) + } + if len(def.Tools) != 1 || def.Tools[0]["type"] != "system" { + t.Errorf("tools = %+v", def.Tools) + } + if def.Avatar["character"] != "lisa" { + t.Errorf("avatar = %+v", def.Avatar) + } +} + // TestCreateVoiceAgentAPIRequest_ExplicitManaged verifies that explicitly // setting model_type: managed is accepted (idempotent with the default). func TestCreateVoiceAgentAPIRequest_ExplicitManaged(t *testing.T) { @@ -263,3 +359,18 @@ func TestCreateVoiceAgentAPIRequest_InvalidModelType(t *testing.T) { t.Error("expected error for unsupported model_type") } } + +func TestCreateVoiceAgentAPIRequest_InvalidAdvancedConfig(t *testing.T) { + t.Parallel() + badThreshold := 2.0 + agent := VoiceAgent{ + AgentDefinition: AgentDefinition{Kind: AgentKindPromptVoice, Name: "v"}, + Model: &Model{Id: "gpt-realtime"}, + Audio: &VoiceAudio{Input: &VoiceAudioInput{ + TurnDetection: &VoiceTurnDetection{Type: "server_vad", Threshold: &badThreshold}, + }}, + } + if _, err := CreateVoiceAgentAPIRequest(agent); err == nil { + t.Error("expected error for invalid advanced config") + } +} diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go index d61fed8905e..3edabb4b5da 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go @@ -440,6 +440,7 @@ func ValidateAgentDefinition(templateBytes []byte) error { "template.model_type '%s' is not supported; use '%s' or '%s'", agent.ModelType, VoiceModelTypeManaged, VoiceModelTypeSelfDeployed)) } + errors = append(errors, validateVoiceAgentAdvancedConfig(agent)...) } else { errors = append(errors, fmt.Sprintf("failed to unmarshal to VoiceAgent: %v", err)) } @@ -459,6 +460,71 @@ func ValidateAgentDefinition(templateBytes []byte) error { return nil } +func validateVoiceAgentAdvancedConfig(agent VoiceAgent) []string { + var errors []string + for i, modality := range agent.OutputModalities { + if strings.TrimSpace(modality) == "" { + errors = append(errors, fmt.Sprintf("template.output_modalities[%d] must not be blank", i)) + } + } + + if agent.Audio == nil { + return errors + } + if agent.Audio.Input != nil { + errors = append(errors, validateVoiceAudioFormat("template.audio.input.format", agent.Audio.Input.Format)...) + if nr := agent.Audio.Input.NoiseReduction; nr != nil && strings.TrimSpace(nr.Type) == "" { + errors = append(errors, "template.audio.input.noise_reduction.type must not be blank") + } + if td := agent.Audio.Input.TurnDetection; td != nil { + if strings.TrimSpace(td.Type) == "" { + errors = append(errors, "template.audio.input.turn_detection.type must not be blank") + } + if td.Threshold != nil && (*td.Threshold < 0 || *td.Threshold > 1) { + errors = append(errors, "template.audio.input.turn_detection.threshold must be between 0 and 1") + } + if td.PrefixPaddingMs != nil && *td.PrefixPaddingMs < 0 { + errors = append(errors, "template.audio.input.turn_detection.prefix_padding_ms must be >= 0") + } + if td.SilenceDurationMs != nil && *td.SilenceDurationMs < 0 { + errors = append(errors, "template.audio.input.turn_detection.silence_duration_ms must be >= 0") + } + } + } + if agent.Audio.Output != nil { + errors = append(errors, validateVoiceAudioFormat("template.audio.output.format", agent.Audio.Output.Format)...) + if voice := agent.Audio.Output.Voice; voice != nil { + if strings.TrimSpace(voice.Type) == "" { + errors = append(errors, "template.audio.output.voice.type must not be blank") + } + if strings.TrimSpace(voice.Name) == "" { + errors = append(errors, "template.audio.output.voice.name must not be blank") + } + } + if speed := agent.Audio.Output.Speed; speed != nil && (*speed < 0.25 || *speed > 1.5) { + errors = append(errors, "template.audio.output.speed must be between 0.25 and 1.5") + } + } + return errors +} + +func validateVoiceAudioFormat(path string, format *VoiceAudioFormat) []string { + if format == nil { + return nil + } + var errors []string + formatType := strings.TrimSpace(format.Type) + if formatType == "" { + errors = append(errors, path+".type must not be blank") + } else if formatType != "audio/pcm" && formatType != "audio/pcmu" && formatType != "audio/pcma" { + errors = append(errors, path+".type must be 'audio/pcm', 'audio/pcmu', or 'audio/pcma'") + } + if format.Rate != nil && *format.Rate <= 0 { + errors = append(errors, path+".rate must be greater than 0") + } + return errors +} + // Validate that the agent name matches the expected deployable format func ValidateAgentName(name string) error { if name == "" { diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse_voice_test.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse_voice_test.go index 6aba8a368dd..ee726c303ca 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse_voice_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse_voice_test.go @@ -49,6 +49,91 @@ template: } } +func TestExtractAgentDefinition_PromptVoiceAdvancedConfig(t *testing.T) { + yamlContent := []byte(` +name: voice-agent +template: + kind: prompt-voice + name: voice-agent + model: + id: gpt-realtime + structured_inputs: + agent_persona: + type: string + defaultValue: Ada + audio: + input: + format: + type: audio/pcmu + rate: 16000 + noise_reduction: + type: near_field + turn_detection: + type: server_vad + threshold: 0.45 + prefix_padding_ms: 250 + silence_duration_ms: 600 + create_response: false + transcription: + model: whisper-1 + language: en-US + prompt: Contoso terms + output: + format: + type: audio/pcm + rate: 24000 + voice: + type: azure_standard + name: en-US-AvaNeural + style: cheerful + speed: 1.1 + output_modalities: [audio, text] + tools: + - type: system + name: end_conversation + avatar: + type: video-avatar + character: lisa +`) + + agent, err := ExtractAgentDefinition(yamlContent) + if err != nil { + t.Fatalf("ExtractAgentDefinition failed: %v", err) + } + voiceAgent := agent.(VoiceAgent) + if voiceAgent.StructuredInputs["agent_persona"] == nil { + t.Fatalf("StructuredInputs = %+v", voiceAgent.StructuredInputs) + } + if voiceAgent.Audio == nil || voiceAgent.Audio.Input == nil || voiceAgent.Audio.Output == nil { + t.Fatalf("Audio = %+v", voiceAgent.Audio) + } + if voiceAgent.Audio.Input.Format.Type != "audio/pcmu" || *voiceAgent.Audio.Input.Format.Rate != 16000 { + t.Errorf("input format = %+v", voiceAgent.Audio.Input.Format) + } + if voiceAgent.Audio.Input.NoiseReduction.Type != "near_field" { + t.Errorf("noise reduction = %+v", voiceAgent.Audio.Input.NoiseReduction) + } + if voiceAgent.Audio.Input.TurnDetection.CreateResponse == nil || *voiceAgent.Audio.Input.TurnDetection.CreateResponse { + t.Errorf("turn detection = %+v", voiceAgent.Audio.Input.TurnDetection) + } + if voiceAgent.Audio.Input.Transcription.Language == nil || *voiceAgent.Audio.Input.Transcription.Language != "en-US" { + t.Errorf("transcription = %+v", voiceAgent.Audio.Input.Transcription) + } + if voiceAgent.Audio.Output.Voice.Name != "en-US-AvaNeural" || + voiceAgent.Audio.Output.Voice.Style == nil || *voiceAgent.Audio.Output.Voice.Style != "cheerful" { + t.Errorf("output voice = %+v", voiceAgent.Audio.Output.Voice) + } + if len(voiceAgent.OutputModalities) != 2 || voiceAgent.OutputModalities[1] != "text" { + t.Errorf("OutputModalities = %v", voiceAgent.OutputModalities) + } + if len(voiceAgent.Tools) != 1 || voiceAgent.Tools[0]["type"] != "system" { + t.Errorf("Tools = %+v", voiceAgent.Tools) + } + if voiceAgent.Avatar["character"] != "lisa" { + t.Errorf("Avatar = %+v", voiceAgent.Avatar) + } +} + // TestValidateAgentDefinition_PromptVoice_OK validates a minimal well-formed // prompt-voice manifest. func TestValidateAgentDefinition_PromptVoice_OK(t *testing.T) { @@ -118,3 +203,53 @@ model_type: unsupported t.Fatalf("expected invalid model_type error, got: %v", err) } } + +func TestValidateAgentDefinition_PromptVoice_InvalidAdvancedConfig(t *testing.T) { + yamlContent := []byte(` +kind: prompt-voice +name: voice-agent +model: + id: gpt-realtime +audio: + input: + format: + type: "" + rate: 0 + turn_detection: + type: server_vad + threshold: 2 + output: + speed: 2 +`) + err := ValidateAgentDefinition(yamlContent) + if err == nil { + t.Fatal("expected advanced config validation error") + } + for _, want := range []string{"format.type", "format.rate", "threshold", "speed"} { + if !strings.Contains(err.Error(), want) { + t.Fatalf("expected error to contain %q, got: %v", want, err) + } + } +} + +func TestValidateAgentDefinition_PromptVoice_UnsupportedAudioFormat(t *testing.T) { + yamlContent := []byte(` +kind: prompt-voice +name: voice-agent +model: + id: gpt-realtime +audio: + input: + format: + type: audio/opus + rate: 24000 +`) + err := ValidateAgentDefinition(yamlContent) + if err == nil { + t.Fatal("expected unsupported audio format validation error") + } + if !strings.Contains(err.Error(), "audio/pcm") || !strings.Contains(err.Error(), "audio/pcmu") || + !strings.Contains(err.Error(), "audio/pcma") { + t.Fatalf("expected supported audio formats in error, got: %v", err) + } +} diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/yaml.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/yaml.go index fd70eb6aa45..a240e537e9b 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/yaml.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/yaml.go @@ -214,9 +214,79 @@ type VoiceAgent struct { // Voice is the output voice name (e.g. "en-US-Ava:DragonHDLatestNeural" for // an Azure Neural voice, or "alloy" for an OpenAI realtime voice). Voice *string `json:"voice,omitempty" yaml:"voice,omitempty"` + // StructuredInputs declares template inputs used by voice instructions. + StructuredInputs map[string]any `json:"structuredInputs,omitempty" yaml:"structured_inputs,omitempty"` + // Audio customizes the input and output voice pipeline. Missing fields keep azd defaults. + Audio *VoiceAudio `json:"audio,omitempty" yaml:"audio,omitempty"` + // OutputModalities declares response modalities such as audio or text. + OutputModalities []string `json:"outputModalities,omitempty" yaml:"output_modalities,omitempty"` // Store toggles server-side logging (transcript + per-turn audio). Optional; // the service defaults to false when omitted. Store *bool `json:"store,omitempty" yaml:"store,omitempty"` + // Tools are passed through to the prompt voice service. Supported service-side + // tool types include function, mcp, system, and toolbox. + Tools []map[string]any `json:"tools,omitempty" yaml:"tools,omitempty"` + // Avatar customizes voice avatar output for services that support it. + Avatar map[string]any `json:"avatar,omitempty" yaml:"avatar,omitempty"` +} + +// VoiceAudio bundles optional prompt voice input/output audio overrides. +type VoiceAudio struct { + Input *VoiceAudioInput `json:"input,omitempty" yaml:"input,omitempty"` + Output *VoiceAudioOutput `json:"output,omitempty" yaml:"output,omitempty"` +} + +// VoiceAudioInput customizes caller-to-agent audio. +type VoiceAudioInput struct { + Format *VoiceAudioFormat `json:"format,omitempty" yaml:"format,omitempty"` + NoiseReduction *VoiceNoiseReduction `json:"noiseReduction,omitempty" yaml:"noise_reduction,omitempty"` + TurnDetection *VoiceTurnDetection `json:"turnDetection,omitempty" yaml:"turn_detection,omitempty"` + Transcription *VoiceTranscription `json:"transcription,omitempty" yaml:"transcription,omitempty"` +} + +// VoiceAudioOutput customizes agent-to-caller audio. +type VoiceAudioOutput struct { + Format *VoiceAudioFormat `json:"format,omitempty" yaml:"format,omitempty"` + Voice *VoiceConfig `json:"voice,omitempty" yaml:"voice,omitempty"` + Speed *float64 `json:"speed,omitempty" yaml:"speed,omitempty"` +} + +// VoiceAudioFormat describes an audio stream format. +type VoiceAudioFormat struct { + Type string `json:"type" yaml:"type"` + Rate *int `json:"rate,omitempty" yaml:"rate,omitempty"` +} + +// VoiceNoiseReduction configures input audio noise reduction. +type VoiceNoiseReduction struct { + Type string `json:"type" yaml:"type"` +} + +// VoiceTurnDetection configures server-side turn detection. +type VoiceTurnDetection struct { + Type string `json:"type" yaml:"type"` + Threshold *float64 `json:"threshold,omitempty" yaml:"threshold,omitempty"` + PrefixPaddingMs *int `json:"prefixPaddingMs,omitempty" yaml:"prefix_padding_ms,omitempty"` + SilenceDurationMs *int `json:"silenceDurationMs,omitempty" yaml:"silence_duration_ms,omitempty"` + CreateResponse *bool `json:"createResponse,omitempty" yaml:"create_response,omitempty"` + Eagerness *string `json:"eagerness,omitempty" yaml:"eagerness,omitempty"` +} + +// VoiceTranscription configures input transcription. +type VoiceTranscription struct { + Model string `json:"model,omitempty" yaml:"model,omitempty"` + Language *string `json:"language,omitempty" yaml:"language,omitempty"` + Prompt *string `json:"prompt,omitempty" yaml:"prompt,omitempty"` +} + +// VoiceConfig selects the output voice. +type VoiceConfig struct { + Type string `json:"type" yaml:"type"` + Name string `json:"name" yaml:"name"` + Style *string `json:"style,omitempty" yaml:"style,omitempty"` + Pitch *string `json:"pitch,omitempty" yaml:"pitch,omitempty"` + Rate *string `json:"rate,omitempty" yaml:"rate,omitempty"` + Locale *string `json:"locale,omitempty" yaml:"locale,omitempty"` } // ContainerResources represents the resource allocation for a containerized agent. diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/agent_definition.go b/cli/azd/extensions/azure.ai.agents/internal/project/agent_definition.go index 26b5a06f874..b2c06cee9c2 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/agent_definition.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/agent_definition.go @@ -138,35 +138,50 @@ type AgentDefinitionInline struct { // Voice-agent fields (kind: prompt-voice). All omitempty so container/ // workflow entries are byte-for-byte unchanged. - ModelType agent_yaml.VoiceModelType `json:"modelType,omitempty"` - Model *agent_yaml.Model `json:"model,omitempty"` - Instructions *string `json:"instructions,omitempty"` - Voice *string `json:"voice,omitempty"` - Store *bool `json:"store,omitempty"` + ModelType agent_yaml.VoiceModelType `json:"modelType,omitempty"` + Model *agent_yaml.Model `json:"model,omitempty"` + Instructions *string `json:"instructions,omitempty"` + Voice *string `json:"voice,omitempty"` + StructuredInputs map[string]any `json:"structuredInputs,omitempty"` + Audio *agent_yaml.VoiceAudio `json:"audio,omitempty"` + OutputModalities []string `json:"outputModalities,omitempty"` + Store *bool `json:"store,omitempty"` + Tools []map[string]any `json:"tools,omitempty"` + Avatar map[string]any `json:"avatar,omitempty"` } // voiceAgentDefinitionToInline projects a VoiceAgent into the inline definition // written to azure.yaml. Voice agents carry no container/image/code config. func voiceAgentDefinitionToInline(va agent_yaml.VoiceAgent) AgentDefinitionInline { return AgentDefinitionInline{ - AgentDefinition: va.AgentDefinition, - ModelType: va.ModelType, - Model: va.Model, - Instructions: va.Instructions, - Voice: va.Voice, - Store: va.Store, + AgentDefinition: va.AgentDefinition, + ModelType: va.ModelType, + Model: va.Model, + Instructions: va.Instructions, + Voice: va.Voice, + StructuredInputs: va.StructuredInputs, + Audio: va.Audio, + OutputModalities: va.OutputModalities, + Store: va.Store, + Tools: va.Tools, + Avatar: va.Avatar, } } // toVoiceAgent rebuilds an agent_yaml.VoiceAgent from the inline definition. func (d AgentDefinitionInline) toVoiceAgent() agent_yaml.VoiceAgent { return agent_yaml.VoiceAgent{ - AgentDefinition: d.AgentDefinition, - ModelType: d.ModelType, - Model: d.Model, - Instructions: d.Instructions, - Voice: d.Voice, - Store: d.Store, + AgentDefinition: d.AgentDefinition, + ModelType: d.ModelType, + Model: d.Model, + Instructions: d.Instructions, + Voice: d.Voice, + StructuredInputs: d.StructuredInputs, + Audio: d.Audio, + OutputModalities: d.OutputModalities, + Store: d.Store, + Tools: d.Tools, + Avatar: d.Avatar, } } diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go index 9e48effc3c0..0606a15095d 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go @@ -34,16 +34,26 @@ func TestVoiceAgentInlineServicePropertiesRoundTrip_BYOM(t *testing.T) { instructions := "Route callers to the right team." voice := "alloy" store := true + rate := 16000 + speed := 1.1 props, err := VoiceAgentDefinitionToServiceProperties(agent_yaml.VoiceAgent{ AgentDefinition: agent_yaml.AgentDefinition{ Kind: agent_yaml.AgentKindPromptVoice, Name: "voice-agent", }, - ModelType: agent_yaml.VoiceModelTypeSelfDeployed, - Model: &agent_yaml.Model{Id: "my-realtime-deployment"}, - Instructions: &instructions, - Voice: &voice, - Store: &store, + ModelType: agent_yaml.VoiceModelTypeSelfDeployed, + Model: &agent_yaml.Model{Id: "my-realtime-deployment"}, + Instructions: &instructions, + Voice: &voice, + StructuredInputs: map[string]any{"persona": map[string]any{"type": "string"}}, + Audio: &agent_yaml.VoiceAudio{Output: &agent_yaml.VoiceAudioOutput{ + Format: &agent_yaml.VoiceAudioFormat{Type: "audio/pcm", Rate: &rate}, + Speed: &speed, + }}, + OutputModalities: []string{"audio", "text"}, + Store: &store, + Tools: []map[string]any{{"type": "system", "name": "end_conversation"}}, + Avatar: map[string]any{"type": "video-avatar", "character": "lisa"}, }, nil) require.NoError(t, err) @@ -64,8 +74,20 @@ func TestVoiceAgentInlineServicePropertiesRoundTrip_BYOM(t *testing.T) { require.Equal(t, instructions, *got.Instructions) require.NotNil(t, got.Voice) require.Equal(t, voice, *got.Voice) + require.NotNil(t, got.StructuredInputs["persona"]) + require.NotNil(t, got.Audio) + require.NotNil(t, got.Audio.Output) + require.Equal(t, "audio/pcm", got.Audio.Output.Format.Type) + require.NotNil(t, got.Audio.Output.Format.Rate) + require.Equal(t, rate, *got.Audio.Output.Format.Rate) + require.NotNil(t, got.Audio.Output.Speed) + require.Equal(t, speed, *got.Audio.Output.Speed) + require.Equal(t, []string{"audio", "text"}, got.OutputModalities) require.NotNil(t, got.Store) require.Equal(t, store, *got.Store) + require.Len(t, got.Tools, 1) + require.Equal(t, "system", got.Tools[0]["type"]) + require.Equal(t, "lisa", got.Avatar["character"]) } func TestApplyAgentMetadata(t *testing.T) { diff --git a/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json b/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json index e486a47de8b..8f5991ac5d0 100644 --- a/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json +++ b/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json @@ -64,14 +64,45 @@ "type": "string", "description": "Voice agent (kind: prompt-voice) system prompt for the assistant." }, - "voice": { - "type": "string", - "description": "Voice agent (kind: prompt-voice) output voice name (e.g. 'en-US-Ava:DragonHDLatestNeural' for an Azure Neural voice, or 'alloy' for an OpenAI realtime voice)." - }, - "store": { - "type": "boolean", - "description": "Voice agent (kind: prompt-voice) server-side logging toggle (transcript + per-turn audio). Defaults to false when omitted." - }, + "voice": { + "type": "string", + "description": "Voice agent (kind: prompt-voice) output voice name (e.g. 'en-US-Ava:DragonHDLatestNeural' for an Azure Neural voice, or 'alloy' for an OpenAI realtime voice)." + }, + "structuredInputs": { + "type": "object", + "description": "Voice agent (kind: prompt-voice) structured prompt inputs used by templated instructions and tool argument bindings. Each value follows the prompt agent StructuredInputDefinition shape: description, defaultValue, schema, required.", + "additionalProperties": true + }, + "audio": { + "$ref": "#/definitions/VoiceAudio" + }, + "outputModalities": { + "type": "array", + "description": "Voice agent (kind: prompt-voice) output modalities. Well-known values are audio, text, animation, and avatar; the service may add extension values.", + "items": { "type": "string", "minLength": 1 } + }, + "store": { + "type": "boolean", + "description": "Voice agent (kind: prompt-voice) server-side logging toggle (transcript + per-turn audio). Defaults to false when omitted." + }, + "tools": { + "type": "array", + "description": "Voice agent (kind: prompt-voice) tools. Directly supported voice tool types are function, mcp, system, and toolbox. Server-side tools such as web_search, azure_ai_search, and openapi must be packaged in a toolbox.", + "items": { "type": "object", "additionalProperties": true } + }, + "avatar": { + "type": "object", + "description": "Voice agent (kind: prompt-voice) avatar output configuration.", + "properties": { + "type": { "type": "string", "description": "Avatar type. Well-known values are video-avatar and photo-avatar; the service may support extension values." }, + "character": { "type": "string", "description": "Avatar character identifier, such as lisa. Required by the service when avatar is configured." }, + "style": { "type": "string", "description": "Avatar style, such as casual-sitting." }, + "customized": { "type": "boolean", "description": "Whether a customized avatar is used." }, + "output_protocol": { "type": "string", "description": "Avatar video output protocol. Well-known values are webrtc and websocket." } + }, + "required": ["type", "character"], + "additionalProperties": true + }, "name": { "type": "string", "description": "The agent name." @@ -152,20 +183,100 @@ "required": ["protocol"], "additionalProperties": false }, - "CodeConfiguration": { + "CodeConfiguration": { "type": "object", "description": "Code deploy configuration. When present, the agent is deployed from source (ZIP) instead of a container image.", "properties": { "runtime": { "type": "string", "description": "Runtime identifier (e.g., 'python_3_12', 'dotnet_9')." }, "entryPoint": { "type": "string", "description": "Entry point for the agent source." }, "dependencyResolution": { "type": "string", "description": "Dependency resolution mode (e.g., 'bundled', 'remote_build')." } - }, - "required": ["runtime", "entryPoint"], - "additionalProperties": false - }, - "SessionConfiguration": { - "type": "object", - "description": "Optional hosted-agent session runtime settings. When omitted, the service applies its defaults (idle timeout 900 seconds).", + }, + "required": ["runtime", "entryPoint"], + "additionalProperties": false + }, + "VoiceAudio": { + "type": "object", + "description": "Prompt voice input and output audio configuration, matching the Voice Live / OpenAI Realtime session schema where supported. Missing fields keep azd defaults.", + "properties": { + "input": { "$ref": "#/definitions/VoiceAudioInput" }, + "output": { "$ref": "#/definitions/VoiceAudioOutput" } + }, + "additionalProperties": false + }, + "VoiceAudioInput": { + "type": "object", + "properties": { + "format": { "$ref": "#/definitions/VoiceAudioFormat" }, + "noiseReduction": { "$ref": "#/definitions/VoiceNoiseReduction" }, + "turnDetection": { "$ref": "#/definitions/VoiceTurnDetection" }, + "transcription": { "$ref": "#/definitions/VoiceTranscription" } + }, + "additionalProperties": false + }, + "VoiceAudioOutput": { + "type": "object", + "properties": { + "format": { "$ref": "#/definitions/VoiceAudioFormat" }, + "voice": { "$ref": "#/definitions/VoiceConfig" }, + "speed": { "type": "number", "minimum": 0.25, "maximum": 1.5, "description": "Output speech speed multiplier. The service accepts 0.25 through 1.5 and defaults to 1 when omitted." } + }, + "additionalProperties": false + }, + "VoiceAudioFormat": { + "type": "object", + "properties": { + "type": { "type": "string", "enum": ["audio/pcm", "audio/pcmu", "audio/pcma"], "description": "Audio format type. Supported values are audio/pcm, audio/pcmu, and audio/pcma." }, + "rate": { "type": "integer", "minimum": 1, "description": "Sample rate in Hz when applicable. Use 24000 for PCM examples; omit for telephony G.711 formats when the service defaults are sufficient." } + }, + "required": ["type"], + "additionalProperties": false + }, + "VoiceNoiseReduction": { + "type": "object", + "properties": { + "type": { "type": "string", "description": "Noise reduction type. Well-known values are near_field, far_field, and azure_deep_noise_suppression." } + }, + "required": ["type"], + "additionalProperties": false + }, + "VoiceTurnDetection": { + "type": "object", + "properties": { + "type": { "type": "string", "description": "Turn detection type. server_vad uses threshold/padding/silence settings; semantic_vad uses eagerness." }, + "threshold": { "type": "number", "minimum": 0, "maximum": 1 }, + "prefixPaddingMs": { "type": "integer", "minimum": 0 }, + "silenceDurationMs": { "type": "integer", "minimum": 0 }, + "createResponse": { "type": "boolean" }, + "eagerness": { "type": "string", "description": "Semantic VAD eagerness. Well-known values are auto, low, medium, and high." } + }, + "required": ["type"], + "additionalProperties": false + }, + "VoiceTranscription": { + "type": "object", + "properties": { + "model": { "type": "string", "description": "Transcription model, such as azure-speech or whisper-1." }, + "language": { "type": "string", "description": "Expected input language as a BCP-47 code, such as en-US. Auto-detected when omitted." }, + "prompt": { "type": "string", "description": "Optional transcription prompt for domain terms or spelling hints." } + }, + "additionalProperties": false + }, + "VoiceConfig": { + "type": "object", + "properties": { + "type": { "type": "string", "description": "Voice provider type, such as openai or azure_standard." }, + "name": { "type": "string", "description": "Voice name. Examples: alloy for OpenAI realtime voices, en-US-AvaNeural for Azure voices." }, + "style": { "type": "string", "description": "Optional Azure voice style, such as cheerful." }, + "pitch": { "type": "string", "description": "Optional Azure voice pitch adjustment." }, + "rate": { "type": "string", "description": "Optional Azure voice speaking rate adjustment." }, + "locale": { "type": "string", "description": "Optional Azure voice locale." } + }, + "required": ["type", "name"], + "additionalProperties": false + }, + "SessionConfiguration": { + "type": "object", + "description": "Optional hosted-agent session runtime settings. When omitted, the service applies its defaults (idle timeout 900 seconds).", "properties": { "idleTimeoutSeconds": { "type": "integer", @@ -173,11 +284,11 @@ "minimum": 300, "maximum": 3600 } - }, - "additionalProperties": false - }, - "Policy": { - "type": "object", + }, + "additionalProperties": false + }, + "Policy": { + "type": "object", "description": "A safety or governance policy attached to the agent.", "properties": { "type": { "type": "string", "description": "Policy type (e.g., 'rai_policy')." },