Support advanced prompt voice agent configuration

cli/azd/extensions/azure.ai.agents/README.md

@@ -133,6 +133,104 @@ Details:
 > the other inline agent properties such as `codeConfiguration` and
 > `environmentVariables`.
 
+## Prompt voice agent configuration
+
+Prompt voice agents keep their editable definition on the `azure.ai.agent`
+service entry in `azure.yaml`. A minimal managed model agent only needs `kind`,
+`model`, and `name`; advanced voice settings can be layered on the same service
+without changing hosted-agent projects. The shape below follows the prompt voice
+service contract used by the current samples and the Vienna implementation; azd
+keeps common fields strongly typed and passes extensible tool/avatar details to
+the service for final validation.
+
+```yaml
+services:
+  voice-agent:
+    host: azure.ai.agent
+    project: src/voice-agent
+    kind: prompt-voice
+    name: voice-agent
+    modelType: managed # or self_deployed for BYOM
+    model:
+      id: gpt-realtime
+    instructions: You are {{agent_persona}}, a concise support agent.
+    structuredInputs:
+      agent_persona:
+        type: string
+        defaultValue: Ada
+    audio:
+      input:
+        format:
+          type: audio/pcm
+          rate: 24000
+        noiseReduction:
+          type: near_field
+        turnDetection:
+          type: server_vad
+          threshold: 0.5
+          prefixPaddingMs: 300
+          silenceDurationMs: 500
+          createResponse: true
+        transcription:
+          model: whisper-1
+          language: en-US
+          prompt: Contoso product names
+      output:
+        format:
+          type: audio/pcm
+          rate: 24000
+        voice:
+          type: azure_standard
+          name: en-US-AvaNeural
+          style: cheerful
+        speed: 1.0
+    outputModalities: [audio]
+    store: true
+    tools:
+      - type: system
+        name: end_conversation
+      - type: function
+        name: get_weather
+        description: Get weather for a city
+        parameters:
+          type: object
+          properties: {}
+    avatar:
+      type: video-avatar
+      character: lisa
+      style: casual-sitting
+      output_protocol: webrtc
+```
+
+Details:
+
+- Existing simple fields continue to work: `instructions`, `voice`, and `store`
+  are shorthand for the common settings. If `audio.output.voice` is present, it
+  takes precedence over the shorthand `voice` field.
+- Missing `audio` fields keep azd defaults: PCM audio at 24 kHz, server VAD,
+  `azure-speech` transcription, and the default Azure Neural voice.
+- Supported `audio.format.type` values are `audio/pcm`, `audio/pcmu`, and
+  `audio/pcma`; `audio.output.speed` must be between `0.25` and `1.5`.
+- `turnDetection.type` supports `server_vad` with `threshold`,
+  `prefixPaddingMs`, `silenceDurationMs`, and `createResponse`, or
+  `semantic_vad` with `eagerness` (`auto`, `low`, `medium`, or `high`).
+- `outputModalities` well-known values are `audio`, `text`, `animation`, and
+  `avatar`. The service defaults to audio when omitted.
+- `structuredInputs` follows the prompt agent structured input shape:
+  `description`, `defaultValue`, `schema`, and `required`. azd converts
+  `defaultValue` to the service wire field `default_value` when deploying.
+- Direct voice tool types are `function`, `mcp`, `system`, and `toolbox`.
+  Server-side tools such as `web_search`, `azure_ai_search`, and `openapi` must
+  be packaged in a toolbox instead of listed directly on the voice agent.
+- `avatar` well-known fields are `type`, `character`, `style`, `customized`, and
+  `output_protocol`; `type` and `character` are required by the service when an
+  avatar is configured. Well-known avatar protocols are `webrtc` and `websocket`.
+- `tools` and `avatar` intentionally remain light pass-through blocks so new
+  service-side capabilities can be adopted without adding azd command flags.
+- In deprecated standalone `agent.yaml` files the equivalent keys use snake_case,
+  for example `model_type`, `structured_inputs`, `output_modalities`,
+  `turn_detection`, and `prefix_padding_ms`.
+
 ## Session idle timeout
 
 A hosted agent's runtime session sandbox is suspended by Foundry after a period

cli/azd/extensions/azure.ai.agents/cspell.yaml

@@ -11,6 +11,9 @@ words:
   # Voice (prompt-voice) agents
   - BYOM
   - Nanami
+  - pcma
+  - pcmu
+  - webrtc
   # Azure region names
   - australiaeast
   - brazilsouth

cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go

@@ -312,7 +312,7 @@ const (
 // VoiceAudioFormat describes a PCM audio stream format (e.g. audio/pcm @ 24 kHz).
 type VoiceAudioFormat struct {
 	Type string `json:"type"`
-	Rate int    `json:"rate"`
+	Rate *int   `json:"rate,omitempty"`
 }
 
 // VoiceTurnDetection configures server-side voice-activity detection so the
@@ -322,32 +322,47 @@ type VoiceTurnDetection struct {
 	Threshold         *float64 `json:"threshold,omitempty"`
 	PrefixPaddingMs   *int     `json:"prefix_padding_ms,omitempty"`
 	SilenceDurationMs *int     `json:"silence_duration_ms,omitempty"`
+	CreateResponse    *bool    `json:"create_response,omitempty"`
+	Eagerness         *string  `json:"eagerness,omitempty"`
 }
 
 // VoiceTranscription enables user-speech transcription events on the input stream.
 type VoiceTranscription struct {
-	Model string `json:"model,omitempty"`
+	Model    string  `json:"model,omitempty"`
+	Language *string `json:"language,omitempty"`
+	Prompt   *string `json:"prompt,omitempty"`
+}
+
+// VoiceNoiseReduction configures input audio noise reduction.
+type VoiceNoiseReduction struct {
+	Type string `json:"type"`
 }
 
 // VoiceInputConfig is the input (caller -> agent) audio configuration.
 type VoiceInputConfig struct {
-	Format        *VoiceAudioFormat   `json:"format,omitempty"`
-	TurnDetection *VoiceTurnDetection `json:"turn_detection,omitempty"`
-	Transcription *VoiceTranscription `json:"transcription,omitempty"`
+	Format         *VoiceAudioFormat    `json:"format,omitempty"`
+	NoiseReduction *VoiceNoiseReduction `json:"noise_reduction,omitempty"`
+	TurnDetection  *VoiceTurnDetection  `json:"turn_detection,omitempty"`
+	Transcription  *VoiceTranscription  `json:"transcription,omitempty"`
 }
 
 // VoiceConfig selects the output voice. Type is "openai" for realtime voices
 // (single lowercase word, e.g. "alloy") or "azure_standard" for Azure Neural
 // voices (e.g. "en-US-Ava:DragonHDLatestNeural").
 type VoiceConfig struct {
-	Type string `json:"type"`
-	Name string `json:"name"`
+	Type   string  `json:"type"`
+	Name   string  `json:"name"`
+	Style  *string `json:"style,omitempty"`
+	Pitch  *string `json:"pitch,omitempty"`
+	Rate   *string `json:"rate,omitempty"`
+	Locale *string `json:"locale,omitempty"`
 }
 
 // VoiceOutputConfig is the output (agent -> caller) audio configuration.
 type VoiceOutputConfig struct {
 	Format *VoiceAudioFormat `json:"format,omitempty"`
 	Voice  *VoiceConfig      `json:"voice,omitempty"`
+	Speed  *float64          `json:"speed,omitempty"`
 }
 
 // VoiceAudioConfig bundles the input and output audio configuration.
@@ -364,9 +379,12 @@ type VoiceAgentDefinition struct {
 	ModelType        VoiceModelType    `json:"model_type"`
 	Model            string            `json:"model"`
 	Instructions     string            `json:"instructions,omitempty"`
+	StructuredInputs map[string]any    `json:"structured_inputs,omitempty"`
 	Audio            *VoiceAudioConfig `json:"audio,omitempty"`
 	OutputModalities []string          `json:"output_modalities,omitempty"`
 	Store            *bool             `json:"store,omitempty"`
+	Tools            []map[string]any  `json:"tools,omitempty"`
+	Avatar           map[string]any    `json:"avatar,omitempty"`
 }
 
 // CreateAgentVersionRequest represents a request to create an agent version

cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go

@@ -553,6 +553,102 @@ func buildVoiceConfig(name string) *agent_api.VoiceConfig {
 	return &agent_api.VoiceConfig{Type: "azure_standard", Name: trimmed}
 }
 
+func defaultVoiceAudioFormat() *agent_api.VoiceAudioFormat {
+	rate := defaultVoiceAudioRate
+	return &agent_api.VoiceAudioFormat{Type: defaultVoiceAudioType, Rate: &rate}
+}
+
+func mapVoiceAudioFormat(format *VoiceAudioFormat, fallback *agent_api.VoiceAudioFormat) *agent_api.VoiceAudioFormat {
+	out := &agent_api.VoiceAudioFormat{}
+	if fallback != nil {
+		*out = *fallback
+	}
+	if format != nil {
+		if strings.TrimSpace(format.Type) != "" {
+			out.Type = strings.TrimSpace(format.Type)
+		}
+		if format.Rate != nil {
+			out.Rate = format.Rate
+		}
+	}
+	return out
+}
+
+func mapVoiceTurnDetection(turnDetection *VoiceTurnDetection) *agent_api.VoiceTurnDetection {
+	out := &agent_api.VoiceTurnDetection{Type: defaultVoiceTurnDetectionType}
+	if turnDetection == nil {
+		return out
+	}
+	if strings.TrimSpace(turnDetection.Type) != "" {
+		out.Type = strings.TrimSpace(turnDetection.Type)
+	}
+	out.Threshold = turnDetection.Threshold
+	out.PrefixPaddingMs = turnDetection.PrefixPaddingMs
+	out.SilenceDurationMs = turnDetection.SilenceDurationMs
+	out.CreateResponse = turnDetection.CreateResponse
+	out.Eagerness = turnDetection.Eagerness
+	return out
+}
+
+func mapVoiceTranscription(transcription *VoiceTranscription) *agent_api.VoiceTranscription {
+	out := &agent_api.VoiceTranscription{Model: defaultVoiceInputTranscriptionModel}
+	if transcription == nil {
+		return out
+	}
+	if strings.TrimSpace(transcription.Model) != "" {
+		out.Model = strings.TrimSpace(transcription.Model)
+	}
+	out.Language = transcription.Language
+	out.Prompt = transcription.Prompt
+	return out
+}
+
+func mapVoiceConfig(voice *VoiceConfig, fallbackName string) *agent_api.VoiceConfig {
+	if voice == nil {
+		return buildVoiceConfig(fallbackName)
+	}
+	name := strings.TrimSpace(voice.Name)
+	if name == "" {
+		name = fallbackName
+	}
+	voiceType := strings.TrimSpace(voice.Type)
+	if voiceType == "" {
+		return buildVoiceConfig(name)
+	}
+	return &agent_api.VoiceConfig{
+		Type:   voiceType,
+		Name:   name,
+		Style:  voice.Style,
+		Pitch:  voice.Pitch,
+		Rate:   voice.Rate,
+		Locale: voice.Locale,
+	}
+}
+
+func mapVoiceStructuredInputs(inputs map[string]any) map[string]any {
+	if len(inputs) == 0 {
+		return nil
+	}
+	out := make(map[string]any, len(inputs))
+	for name, input := range inputs {
+		inputMap, ok := input.(map[string]any)
+		if !ok {
+			out[name] = input
+			continue
+		}
+
+		mapped := maps.Clone(inputMap)
+		if value, ok := mapped["defaultValue"]; ok {
+			if _, hasSnakeCase := mapped["default_value"]; !hasSnakeCase {
+				mapped["default_value"] = value
+			}
+			delete(mapped, "defaultValue")
+		}
+		out[name] = mapped
+	}
+	return out
+}
+
 // CreateVoiceAgentAPIRequest builds a CreateAgentRequest for a declarative
 // voice agent. It translates the authoring kind "prompt-voice" into the
 // data-plane service kind "voice" and defaults the audio pipeline.
@@ -574,6 +670,9 @@ func CreateVoiceAgentAPIRequest(voiceAgent VoiceAgent) (*agent_api.CreateAgentRe
 			"model_type '%s' is not supported; use '%s' or '%s'",
 			voiceAgent.ModelType, VoiceModelTypeManaged, VoiceModelTypeSelfDeployed)
 	}
+	if errors := validateVoiceAgentAdvancedConfig(voiceAgent); len(errors) > 0 {
+		return nil, fmt.Errorf("invalid prompt-voice configuration: %s", strings.Join(errors, "; "))
+	}
 
 	instructions := defaultVoiceInstructions
 	if voiceAgent.Instructions != nil && *voiceAgent.Instructions != "" {
@@ -585,32 +684,60 @@ func CreateVoiceAgentAPIRequest(voiceAgent VoiceAgent) (*agent_api.CreateAgentRe
 		voiceName = *voiceAgent.Voice
 	}
 
-	audioFormat := &agent_api.VoiceAudioFormat{
-		Type: defaultVoiceAudioType,
-		Rate: defaultVoiceAudioRate,
+	inputFormat := defaultVoiceAudioFormat()
+	outputFormat := defaultVoiceAudioFormat()
+	turnDetection := mapVoiceTurnDetection(nil)
+	transcription := mapVoiceTranscription(nil)
+	var noiseReduction *agent_api.VoiceNoiseReduction
+	outputVoice := buildVoiceConfig(voiceName)
+	var outputSpeed *float64
+	if voiceAgent.Audio != nil {
+		if voiceAgent.Audio.Input != nil {
+			inputFormat = mapVoiceAudioFormat(voiceAgent.Audio.Input.Format, inputFormat)
+			if voiceAgent.Audio.Input.NoiseReduction != nil {
+				noiseReduction = &agent_api.VoiceNoiseReduction{Type: strings.TrimSpace(voiceAgent.Audio.Input.NoiseReduction.Type)}
+			}
+			turnDetection = mapVoiceTurnDetection(voiceAgent.Audio.Input.TurnDetection)
+			transcription = mapVoiceTranscription(voiceAgent.Audio.Input.Transcription)
+		}
+		if voiceAgent.Audio.Output != nil {
+			outputFormat = mapVoiceAudioFormat(voiceAgent.Audio.Output.Format, outputFormat)
+			outputVoice = mapVoiceConfig(voiceAgent.Audio.Output.Voice, voiceName)
+			outputSpeed = voiceAgent.Audio.Output.Speed
+		}
+	}
+
+	outputModalities := []string{"audio"}
+	if len(voiceAgent.OutputModalities) > 0 {
+		outputModalities = voiceAgent.OutputModalities
 	}
 
 	voiceDef := agent_api.VoiceAgentDefinition{
 		AgentDefinition: agent_api.AgentDefinition{
 			// Translate authoring kind prompt-voice -> service kind voice.
 			Kind: agent_api.AgentKindVoice,
 		},
-		ModelType:    modelType,
-		Model:        modelID,
-		Instructions: instructions,
+		ModelType:        modelType,
+		Model:            modelID,
+		Instructions:     instructions,
+		StructuredInputs: mapVoiceStructuredInputs(voiceAgent.StructuredInputs),
 		Audio: &agent_api.VoiceAudioConfig{
 			Input: &agent_api.VoiceInputConfig{
-				Format:        audioFormat,
-				TurnDetection: &agent_api.VoiceTurnDetection{Type: defaultVoiceTurnDetectionType},
-				Transcription: &agent_api.VoiceTranscription{Model: defaultVoiceInputTranscriptionModel},
+				Format:         inputFormat,
+				NoiseReduction: noiseReduction,
+				TurnDetection:  turnDetection,
+				Transcription:  transcription,
 			},
 			Output: &agent_api.VoiceOutputConfig{
-				Format: audioFormat,
-				Voice:  buildVoiceConfig(voiceName),
+				Format: outputFormat,
+				Voice:  outputVoice,
+				Speed:  outputSpeed,
 			},
 		},
-		OutputModalities: []string{"audio"},
+		OutputModalities: outputModalities,
 		Store:            voiceAgent.Store,
+		Tools:            voiceAgent.Tools,
+		Avatar:           voiceAgent.Avatar,
 	}
 
 	return createAgentAPIRequest(voiceAgent.AgentDefinition, voiceDef, nil, nil)