From 4f42bb6b06191f691599c363c34cb1bf3803b92c Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:33:40 +0800 Subject: [PATCH 01/38] feat(agents): add unified voice api mode --- cli/azd/extensions/azure.ai.agents/README.md | 28 +++++ .../internal/pkg/agents/agent_api/models.go | 29 +++++ .../pkg/agents/agent_api/operations.go | 87 ++++++++----- .../pkg/agents/agent_api/operations_test.go | 44 +++++++ .../internal/pkg/agents/agent_yaml/map.go | 70 ++++++++++- .../pkg/agents/agent_yaml/map_voice_test.go | 50 ++++++++ .../internal/project/service_target_agent.go | 114 +++++++++++++++--- .../project/service_target_agent_test.go | 38 ++++++ 8 files changed, 412 insertions(+), 48 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/README.md b/cli/azd/extensions/azure.ai.agents/README.md index 80df72db3ac..5425af096d5 100644 --- a/cli/azd/extensions/azure.ai.agents/README.md +++ b/cli/azd/extensions/azure.ai.agents/README.md @@ -133,6 +133,34 @@ Details: > the other inline agent properties such as `codeConfiguration` and > `environmentVariables`. +## Prompt voice agent API mode + +Prompt voice agents (`kind: prompt-voice`) use the legacy `/voice_agents` API by +default while the unified `/agents` voice API rolls out across regions. To run +regression tests against the unified API, set `AZURE_VOICE_AGENT_API` before +`azd deploy`: + +```bash +# Default: legacy /voice_agents API +azd env set AZURE_VOICE_AGENT_API legacy + +# Unified /agents API using the current object-shaped audio.output.voice payload +azd env set AZURE_VOICE_AGENT_API unified + +# Unified /agents API using the TiP/spec flat audio.output.voice payload +azd env set AZURE_VOICE_AGENT_API unified-flat +``` + +Details: + +- `legacy` remains the default and preserves existing behavior. +- `unified` and `unified-flat` create voice agents through `/agents`; repeat + deploys update the existing agent and create a new version through + `/agents/{name}` when the agent name is already present in the azd environment. +- `unified-flat` is intended for TiP/new-service validation. Non-TiP regions may + still require `legacy` or `unified` until the flat output shape is fully + rolled out. + ## Session idle timeout A hosted agent's runtime session sandbox is suspended by Foundry after a period diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go index a276ef6559c..96b33806e95 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go @@ -350,12 +350,28 @@ type VoiceOutputConfig struct { Voice *VoiceConfig `json:"voice,omitempty"` } +// VoiceOutputConfigFlat is the newer Voice Live output shape used by the +// unified /agents voice API in TiP. Older regions still accept/return the +// object-shaped VoiceOutputConfig above. +type VoiceOutputConfigFlat struct { + Format *VoiceAudioFormat `json:"format,omitempty"` + Voice string `json:"voice,omitempty"` + VoiceType string `json:"voice_type,omitempty"` + VoiceLocale string `json:"voice_locale,omitempty"` +} + // VoiceAudioConfig bundles the input and output audio configuration. type VoiceAudioConfig struct { Input *VoiceInputConfig `json:"input,omitempty"` Output *VoiceOutputConfig `json:"output,omitempty"` } +// VoiceAudioConfigFlat bundles voice audio config with the flat output shape. +type VoiceAudioConfigFlat struct { + Input *VoiceInputConfig `json:"input,omitempty"` + Output *VoiceOutputConfigFlat `json:"output,omitempty"` +} + // VoiceAgentDefinition is the data-plane definition body POSTed to the // /voice_agents collection for a declarative (managed) voice agent. Its Kind // is always AgentKindVoice ("voice"). @@ -369,6 +385,19 @@ type VoiceAgentDefinition struct { Store *bool `json:"store,omitempty"` } +// VoiceAgentDefinitionFlat is the voice definition shape aligned with the +// unified /agents TiP API, where audio.output.voice is a string and the voice +// provider details are sibling fields. +type VoiceAgentDefinitionFlat struct { + AgentDefinition + ModelType VoiceModelType `json:"model_type"` + Model string `json:"model"` + Instructions string `json:"instructions,omitempty"` + Audio *VoiceAudioConfigFlat `json:"audio,omitempty"` + OutputModalities []string `json:"output_modalities,omitempty"` + Store *bool `json:"store,omitempty"` +} + // CreateAgentVersionRequest represents a request to create an agent version type CreateAgentVersionRequest struct { Description *string `json:"description,omitempty"` diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations.go index c94ebc69294..46c0d9d5e44 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations.go @@ -167,47 +167,24 @@ func (c *AgentClient) CreateAgent(ctx context.Context, request *CreateAgentReque // header while voice agents remain a preview capability. const voiceAgentsPreviewFeature = "VoiceAgents=V1Preview" -// CreateVoiceAgent creates a new declarative (managed) voice agent. -// -// Voice agents live in a separate data-plane collection (/voice_agents), distinct -// from the /agents collection used by hosted/workflow agents. The request -// Definition must be a *VoiceAgentDefinition (service kind "voice"). -// -// overriddenHost, when non-empty, is sent as the x-ms-overridden-host header. -// This routes the request directly to the regional Hyena data-plane host, -// bypassing the public Foundry APIM (whose voice route may not yet be rolled -// out). Pass "" to use the default endpoint routing. -// -// Redeploy semantics: the voice data-plane exposes create-only POST /voice_agents -// with no version/upsert model (unlike hosted agents, which mint a new -// agent-version per deploy). A second `azd deploy` of the same voice service -// therefore re-POSTs with the same name and the service rejects it with a -// non-success status, which this method surfaces as a deploy error rather than -// silently overwriting the existing agent. Idempotent redeploy/update is tracked -// as a follow-up (see the PR "Follow-ups" section); until the service adds an -// update route, redeploy requires deleting the existing voice agent first. -func (c *AgentClient) CreateVoiceAgent( +func (c *AgentClient) doVoiceJSONAgentRequest( ctx context.Context, - request *CreateAgentRequest, - apiVersion string, + method string, + url string, + request any, overriddenHost string, ) (*AgentObject, error) { - url := fmt.Sprintf("%s/voice_agents?api-version=%s", c.endpoint, apiVersion) - payload, err := json.Marshal(request) if err != nil { return nil, fmt.Errorf("failed to marshal request: %w", err) } - req, err := runtime.NewRequest(ctx, http.MethodPost, url) + req, err := runtime.NewRequest(ctx, method, url) if err != nil { return nil, fmt.Errorf("failed to create request: %w", err) } - // Voice agents are a preview feature; the service rejects the request with - // 403 preview_feature_required unless this opt-in header is present. req.Raw().Header.Set("Foundry-Features", voiceAgentsPreviewFeature) - if overriddenHost != "" { req.Raw().Header.Set("x-ms-overridden-host", overriddenHost) } @@ -239,6 +216,60 @@ func (c *AgentClient) CreateVoiceAgent( return &agent, nil } +// CreateVoiceAgent creates a new declarative (managed) voice agent. +// +// Voice agents live in a separate data-plane collection (/voice_agents), distinct +// from the /agents collection used by hosted/workflow agents. The request +// Definition must be a *VoiceAgentDefinition (service kind "voice"). +// +// overriddenHost, when non-empty, is sent as the x-ms-overridden-host header. +// This routes the request directly to the regional Hyena data-plane host, +// bypassing the public Foundry APIM (whose voice route may not yet be rolled +// out). Pass "" to use the default endpoint routing. +// +// Redeploy semantics: the voice data-plane exposes create-only POST /voice_agents +// with no version/upsert model (unlike hosted agents, which mint a new +// agent-version per deploy). A second `azd deploy` of the same voice service +// therefore re-POSTs with the same name and the service rejects it with a +// non-success status, which this method surfaces as a deploy error rather than +// silently overwriting the existing agent. Idempotent redeploy/update is tracked +// as a follow-up (see the PR "Follow-ups" section); until the service adds an +// update route, redeploy requires deleting the existing voice agent first. +func (c *AgentClient) CreateVoiceAgent( + ctx context.Context, + request *CreateAgentRequest, + apiVersion string, + overriddenHost string, +) (*AgentObject, error) { + url := fmt.Sprintf("%s/voice_agents?api-version=%s", c.endpoint, apiVersion) + return c.doVoiceJSONAgentRequest(ctx, http.MethodPost, url, request, overriddenHost) +} + +// CreateVoiceAgentUnified creates a voice agent through the unified /agents +// collection. This path is opt-in while regional rollout is still in progress. +func (c *AgentClient) CreateVoiceAgentUnified( + ctx context.Context, + request *CreateAgentRequest, + apiVersion string, + overriddenHost string, +) (*AgentObject, error) { + url := fmt.Sprintf("%s/agents?api-version=%s", c.endpoint, apiVersion) + return c.doVoiceJSONAgentRequest(ctx, http.MethodPost, url, request, overriddenHost) +} + +// UpdateVoiceAgentUnified creates a new version for an existing voice agent +// through the unified /agents/{name} endpoint. +func (c *AgentClient) UpdateVoiceAgentUnified( + ctx context.Context, + agentName string, + request *UpdateAgentRequest, + apiVersion string, + overriddenHost string, +) (*AgentObject, error) { + url := fmt.Sprintf("%s/agents/%s?api-version=%s", c.endpoint, agentName, apiVersion) + return c.doVoiceJSONAgentRequest(ctx, http.MethodPost, url, request, overriddenHost) +} + // UpdateAgent updates an existing agent func (c *AgentClient) UpdateAgent(ctx context.Context, agentName string, request *UpdateAgentRequest, apiVersion string) (*AgentObject, error) { url := fmt.Sprintf("%s/agents/%s?api-version=%s", c.endpoint, agentName, apiVersion) diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations_test.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations_test.go index 6eb2effae43..cf4d9a116b3 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations_test.go @@ -948,6 +948,50 @@ func TestCreateVoiceAgent_PostsToVoiceCollectionWithPreviewHeader(t *testing.T) require.Contains(t, string(reqBody), `"name":"my-voice"`) } +func TestCreateVoiceAgentUnified_PostsToAgentsWithPreviewHeader(t *testing.T) { + body := `{"object":"agent","id":"va-1","name":"my-voice","versions":{"latest":{}}}` + client, transport := newCaptureClient(http.StatusOK, body) + + agent, err := client.CreateVoiceAgentUnified( + t.Context(), + &CreateAgentRequest{Name: "my-voice"}, + AgentEndpointAPIVersion, + "", + ) + + require.NoError(t, err) + require.Equal(t, "my-voice", agent.Name) + require.Len(t, transport.requests, 1) + req := transport.requests[0] + require.Equal(t, http.MethodPost, req.Method) + require.Equal(t, "/api/projects/proj/agents", req.URL.Path) + require.Equal(t, AgentEndpointAPIVersion, req.URL.Query().Get("api-version")) + require.Equal(t, voiceAgentsPreviewFeature, req.Header.Get("Foundry-Features")) +} + +func TestUpdateVoiceAgentUnified_PostsToNamedAgentWithPreviewHeader(t *testing.T) { + body := `{"object":"agent","id":"va-1","name":"my-voice","versions":{"latest":{"version":"2"}}}` + client, transport := newCaptureClient(http.StatusOK, body) + + agent, err := client.UpdateVoiceAgentUnified( + t.Context(), + "my-voice", + &UpdateAgentRequest{}, + AgentEndpointAPIVersion, + "regional.hyena.example.com", + ) + + require.NoError(t, err) + require.Equal(t, "my-voice", agent.Name) + require.Len(t, transport.requests, 1) + req := transport.requests[0] + require.Equal(t, http.MethodPost, req.Method) + require.Equal(t, "/api/projects/proj/agents/my-voice", req.URL.Path) + require.Equal(t, AgentEndpointAPIVersion, req.URL.Query().Get("api-version")) + require.Equal(t, voiceAgentsPreviewFeature, req.Header.Get("Foundry-Features")) + require.Equal(t, "regional.hyena.example.com", req.Header.Get("x-ms-overridden-host")) +} + func TestCreateVoiceAgent_SetsOverriddenHostHeader(t *testing.T) { client, transport := newCaptureClient(http.StatusCreated, `{"name":"my-voice","versions":{"latest":{}}}`) diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go index 8a385d27632..0404762cb43 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go @@ -553,10 +553,41 @@ func buildVoiceConfig(name string) *agent_api.VoiceConfig { return &agent_api.VoiceConfig{Type: "azure_standard", Name: trimmed} } +func flatVoiceType(voice *agent_api.VoiceConfig) string { + if voice == nil { + return "" + } + if voice.Type == "azure_standard" { + return "azure-standard" + } + return voice.Type +} + +func flatVoiceLocale(voice *agent_api.VoiceConfig) string { + if voice == nil || voice.Name == "" || isOpenAIVoice(voice.Name) { + return "" + } + parts := strings.SplitN(voice.Name, "-", 3) + if len(parts) < 2 { + return "" + } + return parts[0] + "-" + parts[1] +} + // CreateVoiceAgentAPIRequest builds a CreateAgentRequest for a declarative // voice agent. It translates the authoring kind "prompt-voice" into the // data-plane service kind "voice" and defaults the audio pipeline. func CreateVoiceAgentAPIRequest(voiceAgent VoiceAgent) (*agent_api.CreateAgentRequest, error) { + return createVoiceAgentAPIRequest(voiceAgent, false) +} + +// CreateVoiceAgentAPIRequestFlat builds a CreateAgentRequest using the newer +// TiP/unified API flat output voice shape. +func CreateVoiceAgentAPIRequestFlat(voiceAgent VoiceAgent) (*agent_api.CreateAgentRequest, error) { + return createVoiceAgentAPIRequest(voiceAgent, true) +} + +func createVoiceAgentAPIRequest(voiceAgent VoiceAgent, flatOutput bool) (*agent_api.CreateAgentRequest, error) { modelID := "" if voiceAgent.Model != nil { modelID = strings.TrimSpace(voiceAgent.Model.Id) @@ -590,6 +621,37 @@ func CreateVoiceAgentAPIRequest(voiceAgent VoiceAgent) (*agent_api.CreateAgentRe Rate: defaultVoiceAudioRate, } + input := &agent_api.VoiceInputConfig{ + Format: audioFormat, + TurnDetection: &agent_api.VoiceTurnDetection{Type: defaultVoiceTurnDetectionType}, + Transcription: &agent_api.VoiceTranscription{Model: defaultVoiceInputTranscriptionModel}, + } + voiceConfig := buildVoiceConfig(voiceName) + if flatOutput { + voiceDef := agent_api.VoiceAgentDefinitionFlat{ + AgentDefinition: agent_api.AgentDefinition{ + // Translate authoring kind prompt-voice -> service kind voice. + Kind: agent_api.AgentKindVoice, + }, + ModelType: modelType, + Model: modelID, + Instructions: instructions, + Audio: &agent_api.VoiceAudioConfigFlat{ + Input: input, + Output: &agent_api.VoiceOutputConfigFlat{ + Format: audioFormat, + Voice: voiceConfig.Name, + VoiceType: flatVoiceType(voiceConfig), + VoiceLocale: flatVoiceLocale(voiceConfig), + }, + }, + OutputModalities: []string{"audio"}, + Store: voiceAgent.Store, + } + + return createAgentAPIRequest(voiceAgent.AgentDefinition, voiceDef, nil, nil) + } + voiceDef := agent_api.VoiceAgentDefinition{ AgentDefinition: agent_api.AgentDefinition{ // Translate authoring kind prompt-voice -> service kind voice. @@ -599,14 +661,10 @@ func CreateVoiceAgentAPIRequest(voiceAgent VoiceAgent) (*agent_api.CreateAgentRe Model: modelID, Instructions: instructions, Audio: &agent_api.VoiceAudioConfig{ - Input: &agent_api.VoiceInputConfig{ - Format: audioFormat, - TurnDetection: &agent_api.VoiceTurnDetection{Type: defaultVoiceTurnDetectionType}, - Transcription: &agent_api.VoiceTranscription{Model: defaultVoiceInputTranscriptionModel}, - }, + Input: input, Output: &agent_api.VoiceOutputConfig{ Format: audioFormat, - Voice: buildVoiceConfig(voiceName), + Voice: voiceConfig, }, }, OutputModalities: []string{"audio"}, diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go index 137e9b6e847..2ce594e1fbf 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go @@ -172,6 +172,56 @@ func TestCreateVoiceAgentAPIRequest_Overrides(t *testing.T) { } } +func TestCreateVoiceAgentAPIRequestFlat_UsesFlatOutputShape(t *testing.T) { + t.Parallel() + voice := "alloy" + agent := VoiceAgent{ + AgentDefinition: AgentDefinition{Kind: AgentKindPromptVoice, Name: "voice-flat"}, + Model: &Model{Id: "gpt-realtime"}, + Voice: &voice, + } + + req, err := CreateVoiceAgentAPIRequestFlat(agent) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + def := req.Definition.(agent_api.VoiceAgentDefinitionFlat) + if def.Audio.Output.Voice != "alloy" { + t.Errorf("Voice = %q, want alloy", def.Audio.Output.Voice) + } + if def.Audio.Output.VoiceType != "openai" { + t.Errorf("VoiceType = %q, want openai", def.Audio.Output.VoiceType) + } + if def.Audio.Output.VoiceLocale != "" { + t.Errorf("VoiceLocale = %q, want empty", def.Audio.Output.VoiceLocale) + } +} + +func TestCreateVoiceAgentAPIRequestFlat_AzureVoiceLocale(t *testing.T) { + t.Parallel() + voice := "en-US-Ava:DragonHDLatestNeural" + agent := VoiceAgent{ + AgentDefinition: AgentDefinition{Kind: AgentKindPromptVoice, Name: "voice-flat"}, + Model: &Model{Id: "gpt-realtime"}, + Voice: &voice, + } + + req, err := CreateVoiceAgentAPIRequestFlat(agent) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + def := req.Definition.(agent_api.VoiceAgentDefinitionFlat) + if def.Audio.Output.Voice != voice { + t.Errorf("Voice = %q, want %q", def.Audio.Output.Voice, voice) + } + if def.Audio.Output.VoiceType != "azure-standard" { + t.Errorf("VoiceType = %q, want azure-standard", def.Audio.Output.VoiceType) + } + if def.Audio.Output.VoiceLocale != "en-US" { + t.Errorf("VoiceLocale = %q, want en-US", def.Audio.Output.VoiceLocale) + } +} + // TestCreateVoiceAgentAPIRequest_ExplicitManaged verifies that explicitly // setting model_type: managed is accepted (idempotent with the default). func TestCreateVoiceAgentAPIRequest_ExplicitManaged(t *testing.T) { diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go index 3f4039392a2..676b1ecfcbe 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go @@ -2151,12 +2151,50 @@ func (p *AgentServiceTargetProvider) deployHostedAgent( //nolint:gosec // env var key name, not a credential const voiceOverriddenHostEnvKey = "AZURE_VOICE_OVERRIDDEN_HOST" +// voiceAgentAPIEnvKey controls which voice deployment API azd uses. It defaults +// to the legacy /voice_agents path while the unified API rolls out regionally. +// Supported values: +// - legacy: POST /voice_agents with object-shaped audio.output.voice +// - unified: POST /agents or /agents/{name} with object-shaped audio.output.voice +// - unified-flat: POST /agents or /agents/{name} with flat audio.output.voice +// +//nolint:gosec // env var key name, not a credential +const voiceAgentAPIEnvKey = "AZURE_VOICE_AGENT_API" + +type voiceAgentAPIMode string + +const ( + voiceAgentAPIModeLegacy voiceAgentAPIMode = "legacy" + voiceAgentAPIModeUnified voiceAgentAPIMode = "unified" + voiceAgentAPIModeUnifiedFlat voiceAgentAPIMode = "unified-flat" +) + +func resolveVoiceAgentAPIMode(azdEnv map[string]string) (voiceAgentAPIMode, error) { + mode := strings.TrimSpace(azdEnv[voiceAgentAPIEnvKey]) + if mode == "" { + mode = strings.TrimSpace(os.Getenv(voiceAgentAPIEnvKey)) + } + mode = strings.ToLower(strings.ReplaceAll(mode, "_", "-")) + if mode == "" { + return voiceAgentAPIModeLegacy, nil + } + switch voiceAgentAPIMode(mode) { + case voiceAgentAPIModeLegacy, voiceAgentAPIModeUnified, voiceAgentAPIModeUnifiedFlat: + return voiceAgentAPIMode(mode), nil + default: + return "", fmt.Errorf( + "%s must be one of %q, %q, or %q", + voiceAgentAPIEnvKey, voiceAgentAPIModeLegacy, voiceAgentAPIModeUnified, voiceAgentAPIModeUnifiedFlat, + ) + } +} + // deployVoiceAgent deploys a declarative (managed) voice agent (kind: -// prompt-voice) to the Foundry service. Unlike hosted agents, voice agents are -// created synchronously via a single POST to /voice_agents that returns the -// created AgentObject directly — there is no container build, no agent-version -// object, and no active-state polling. This method is intentionally isolated -// from the container deploy path so the two contracts never entangle. +// prompt-voice) to the Foundry service. The legacy /voice_agents API remains +// the default while unified /agents rolls out regionally; AZURE_VOICE_AGENT_API +// can opt into unified modes for regression and TiP validation. This method is +// intentionally isolated from the container deploy path so the two contracts +// never entangle. func (p *AgentServiceTargetProvider) deployVoiceAgent( ctx context.Context, serviceConfig *azdext.ServiceConfig, @@ -2166,7 +2204,21 @@ func (p *AgentServiceTargetProvider) deployVoiceAgent( ) (*azdext.ServiceDeployResult, error) { progress("Deploying voice agent") - request, err := agent_yaml.CreateVoiceAgentAPIRequest(va) + apiMode, err := resolveVoiceAgentAPIMode(azdEnv) + if err != nil { + return nil, exterrors.Validation( + exterrors.CodeInvalidAgentManifest, + err.Error(), + fmt.Sprintf("set %s to legacy, unified, or unified-flat", voiceAgentAPIEnvKey), + ) + } + + var request *agent_api.CreateAgentRequest + if apiMode == voiceAgentAPIModeUnifiedFlat { + request, err = agent_yaml.CreateVoiceAgentAPIRequestFlat(va) + } else { + request, err = agent_yaml.CreateVoiceAgentAPIRequest(va) + } if err != nil { return nil, exterrors.Validation( exterrors.CodeInvalidAgentManifest, @@ -2187,22 +2239,19 @@ func (p *AgentServiceTargetProvider) deployVoiceAgent( agentClient := agent_api.NewAgentClient(projectEndpoint, p.credential) - progress("Creating voice agent") - agentObject, err := agentClient.CreateVoiceAgent( - ctx, request, agent_api.AgentEndpointAPIVersion, azdEnv[voiceOverriddenHostEnvKey], + serviceKey := p.getServiceKey(serviceConfig.Name) + agentObject, err := p.deployVoiceAgentWithMode( + ctx, agentClient, request, apiMode, serviceKey, azdEnv, progress, ) if err != nil { return nil, exterrors.ServiceFromAzure(err, exterrors.OpCreateAgent) } - fmt.Fprintf(os.Stderr, "Voice agent '%s' created successfully!\n", agentObject.Name) + fmt.Fprintf(os.Stderr, "Voice agent '%s' deployed successfully!\n", agentObject.Name) // Persist NAME first and ENDPOINT last. ENDPOINT is used as the voice deploy // completion marker by other commands, so avoid writing it before NAME. - serviceKey := p.getServiceKey(serviceConfig.Name) - baseEndpoint := fmt.Sprintf( - "%s/voice_agents/%s", strings.TrimRight(projectEndpoint, "/"), agentObject.Name, - ) + baseEndpoint := voiceAgentEndpoint(projectEndpoint, agentObject.Name, apiMode) for _, envVar := range []struct{ key, value string }{ {fmt.Sprintf("AGENT_%s_NAME", serviceKey), agentObject.Name}, {fmt.Sprintf("AGENT_%s_ENDPOINT", serviceKey), baseEndpoint}, @@ -2230,6 +2279,43 @@ func (p *AgentServiceTargetProvider) deployVoiceAgent( return &azdext.ServiceDeployResult{Artifacts: artifacts}, nil } +func (p *AgentServiceTargetProvider) deployVoiceAgentWithMode( + ctx context.Context, + agentClient *agent_api.AgentClient, + request *agent_api.CreateAgentRequest, + apiMode voiceAgentAPIMode, + serviceKey string, + azdEnv map[string]string, + progress azdext.ProgressReporter, +) (*agent_api.AgentObject, error) { + overriddenHost := azdEnv[voiceOverriddenHostEnvKey] + if apiMode == voiceAgentAPIModeLegacy { + progress("Creating voice agent using legacy API") + return agentClient.CreateVoiceAgent(ctx, request, agent_api.AgentEndpointAPIVersion, overriddenHost) + } + + if existingName := strings.TrimSpace(azdEnv[fmt.Sprintf("AGENT_%s_NAME", serviceKey)]); existingName != "" { + progress("Updating voice agent using unified API") + updateRequest := &agent_api.UpdateAgentRequest{ + CreateAgentVersionRequest: request.CreateAgentVersionRequest, + } + return agentClient.UpdateVoiceAgentUnified( + ctx, existingName, updateRequest, agent_api.AgentEndpointAPIVersion, overriddenHost, + ) + } + + progress("Creating voice agent using unified API") + return agentClient.CreateVoiceAgentUnified(ctx, request, agent_api.AgentEndpointAPIVersion, overriddenHost) +} + +func voiceAgentEndpoint(projectEndpoint string, agentName string, apiMode voiceAgentAPIMode) string { + trimmedEndpoint := strings.TrimRight(projectEndpoint, "/") + if apiMode == voiceAgentAPIModeLegacy { + return fmt.Sprintf("%s/voice_agents/%s", trimmedEndpoint, agentName) + } + return fmt.Sprintf("%s/agents/%s/endpoint/protocols/voice", trimmedEndpoint, agentName) +} + // packageCodeDeploy creates a ZIP archive of the agent source code, writes it to a temp file, // and computes its SHA-256. Returns the temp file path and SHA-256 hex string. func (p *AgentServiceTargetProvider) packageCodeDeploy(ctx context.Context, serviceConfig *azdext.ServiceConfig) (string, string, error) { diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go index 9e48effc3c0..4f2f230dafc 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go @@ -668,6 +668,44 @@ func TestAdoptServiceConfigIgnoresNilAndKeepsResolvedState(t *testing.T) { require.False(t, provider.serviceConfigResolved) } +func TestResolveVoiceAgentAPIMode_DefaultsToLegacy(t *testing.T) { + t.Setenv(voiceAgentAPIEnvKey, "") + mode, err := resolveVoiceAgentAPIMode(map[string]string{}) + require.NoError(t, err) + require.Equal(t, voiceAgentAPIModeLegacy, mode) +} + +func TestResolveVoiceAgentAPIMode_EnvValues(t *testing.T) { + t.Setenv(voiceAgentAPIEnvKey, "unified_flat") + mode, err := resolveVoiceAgentAPIMode(map[string]string{}) + require.NoError(t, err) + require.Equal(t, voiceAgentAPIModeUnifiedFlat, mode) + + mode, err = resolveVoiceAgentAPIMode(map[string]string{voiceAgentAPIEnvKey: "unified"}) + require.NoError(t, err) + require.Equal(t, voiceAgentAPIModeUnified, mode) +} + +func TestResolveVoiceAgentAPIMode_Invalid(t *testing.T) { + _, err := resolveVoiceAgentAPIMode(map[string]string{voiceAgentAPIEnvKey: "future"}) + require.Error(t, err) + require.Contains(t, err.Error(), voiceAgentAPIEnvKey) +} + +func TestVoiceAgentEndpoint_ByMode(t *testing.T) { + projectEndpoint := "https://proj.services.ai.azure.com/api/projects/p/" + require.Equal( + t, + "https://proj.services.ai.azure.com/api/projects/p/voice_agents/my-agent", + voiceAgentEndpoint(projectEndpoint, "my-agent", voiceAgentAPIModeLegacy), + ) + require.Equal( + t, + "https://proj.services.ai.azure.com/api/projects/p/agents/my-agent/endpoint/protocols/voice", + voiceAgentEndpoint(projectEndpoint, "my-agent", voiceAgentAPIModeUnified), + ) +} + func createSymlinkOrSkip(t *testing.T, oldname, newname string) { t.Helper() From 27073e789f7ca71f63160c32b6b35cd0501cc915 Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:15:42 +0800 Subject: [PATCH 02/38] docs(agents): clarify voice deploy routing comment --- .../internal/project/service_target_agent.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go index 676b1ecfcbe..d240f13545f 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go @@ -1376,13 +1376,11 @@ func (p *AgentServiceTargetProvider) Deploy( return nil, err } - // Voice agents (kind: prompt-voice) use a fundamentally different data-plane - // contract than hosted/workflow agents: a synchronous POST to /voice_agents - // that returns an AgentObject directly, with no version/polling model. Resolve - // the definition first — honoring the AGENT_DEFINITION_PATH override precedence - // so an override drives this dispatch just as it does the container path — and - // route voice to an isolated method so the container deploy path below stays - // byte-for-byte unchanged. + // Voice agents (kind: prompt-voice) use a different data-plane contract than + // hosted/workflow agents. Resolve the definition first — honoring the + // AGENT_DEFINITION_PATH override precedence so an override drives this dispatch + // just as it does the container path — and route voice to an isolated method so + // the container deploy path below stays byte-for-byte unchanged. if isVoice { return p.deployVoiceAgent(ctx, serviceConfig, voiceAgent, azdEnv, progress) } From 16e94ae2bac0d9af3b45b84c145612226e42a85f Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Thu, 20 Aug 2026 10:39:37 +0800 Subject: [PATCH 03/38] fix(agents): harden unified voice deploy --- cli/azd/extensions/azure.ai.agents/README.md | 9 ++-- .../pkg/agents/agent_api/operations.go | 43 +++++++++++++++ .../pkg/agents/agent_api/operations_test.go | 23 ++++++++ .../internal/project/service_target_agent.go | 52 +++++++++++++++---- .../project/service_target_agent_test.go | 14 ++++- 5 files changed, 126 insertions(+), 15 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/README.md b/cli/azd/extensions/azure.ai.agents/README.md index 5425af096d5..6fecb7e17db 100644 --- a/cli/azd/extensions/azure.ai.agents/README.md +++ b/cli/azd/extensions/azure.ai.agents/README.md @@ -154,9 +154,12 @@ azd env set AZURE_VOICE_AGENT_API unified-flat Details: - `legacy` remains the default and preserves existing behavior. -- `unified` and `unified-flat` create voice agents through `/agents`; repeat - deploys update the existing agent and create a new version through - `/agents/{name}` when the agent name is already present in the azd environment. +- `unified` and `unified-flat` check `/agents/{name}` remotely before deploying: + `404` creates through `/agents`, while `200` updates through `/agents/{name}`. +- Unified modes write `AGENT__VERSION` and store the callable voice + WebSocket endpoint as `wss://.../agents/{name}/endpoint/protocols/voice?api-version=v1`. +- `legacy` clears any stale `AGENT__VERSION` value and preserves the + existing `/voice_agents/{name}` endpoint marker. - `unified-flat` is intended for TiP/new-service validation. Non-TiP regions may still require `legacy` or `unified` until the flat output shape is fully rolled out. diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations.go index 46c0d9d5e44..fbd6f514ef3 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations.go @@ -245,6 +245,49 @@ func (c *AgentClient) CreateVoiceAgent( return c.doVoiceJSONAgentRequest(ctx, http.MethodPost, url, request, overriddenHost) } +// GetVoiceAgentUnified retrieves a voice agent through the unified /agents +// endpoint with the voice preview opt-in header. Use this instead of GetAgent +// when deciding whether to create or update a prompt voice agent. +func (c *AgentClient) GetVoiceAgentUnified( + ctx context.Context, + agentName string, + apiVersion string, + overriddenHost string, +) (*AgentObject, error) { + url := fmt.Sprintf("%s/agents/%s?api-version=%s", c.endpoint, agentName, apiVersion) + req, err := runtime.NewRequest(ctx, http.MethodGet, url) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Raw().Header.Set("Foundry-Features", voiceAgentsPreviewFeature) + if overriddenHost != "" { + req.Raw().Header.Set("x-ms-overridden-host", overriddenHost) + } + + resp, err := c.pipeline.Do(req) + if err != nil { + return nil, fmt.Errorf("HTTP request failed: %w", err) + } + defer resp.Body.Close() + + if !runtime.HasStatusCode(resp, http.StatusOK) { + return nil, runtime.NewResponseError(resp) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + + var agent AgentObject + if err := json.Unmarshal(body, &agent); err != nil { + return nil, fmt.Errorf("failed to parse response: %w", err) + } + + return &agent, nil +} + // CreateVoiceAgentUnified creates a voice agent through the unified /agents // collection. This path is opt-in while regional rollout is still in progress. func (c *AgentClient) CreateVoiceAgentUnified( diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations_test.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations_test.go index cf4d9a116b3..7aacd0d7ba5 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations_test.go @@ -969,6 +969,29 @@ func TestCreateVoiceAgentUnified_PostsToAgentsWithPreviewHeader(t *testing.T) { require.Equal(t, voiceAgentsPreviewFeature, req.Header.Get("Foundry-Features")) } +func TestGetVoiceAgentUnified_GetsNamedAgentWithPreviewHeader(t *testing.T) { + body := `{"object":"agent","id":"va-1","name":"my-voice","versions":{"latest":{"version":"3"}}}` + client, transport := newCaptureClient(http.StatusOK, body) + + agent, err := client.GetVoiceAgentUnified( + t.Context(), + "my-voice", + AgentEndpointAPIVersion, + "regional.hyena.example.com", + ) + + require.NoError(t, err) + require.Equal(t, "my-voice", agent.Name) + require.Equal(t, "3", agent.Versions.Latest.Version) + require.Len(t, transport.requests, 1) + req := transport.requests[0] + require.Equal(t, http.MethodGet, req.Method) + require.Equal(t, "/api/projects/proj/agents/my-voice", req.URL.Path) + require.Equal(t, AgentEndpointAPIVersion, req.URL.Query().Get("api-version")) + require.Equal(t, voiceAgentsPreviewFeature, req.Header.Get("Foundry-Features")) + require.Equal(t, "regional.hyena.example.com", req.Header.Get("x-ms-overridden-host")) +} + func TestUpdateVoiceAgentUnified_PostsToNamedAgentWithPreviewHeader(t *testing.T) { body := `{"object":"agent","id":"va-1","name":"my-voice","versions":{"latest":{"version":"2"}}}` client, transport := newCaptureClient(http.StatusOK, body) diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go index d240f13545f..6c0c15c8d15 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go @@ -130,6 +130,19 @@ func buildInvocationsWSProtocolURL(projectEndpoint, agentName string) string { ) } +func buildVoiceWSProtocolURL(projectEndpoint, agentName string) string { + projectEndpoint = strings.TrimSpace(projectEndpoint) + u, err := url.Parse(projectEndpoint) + if err != nil || u.Host == "" { + return "" + } + + return fmt.Sprintf( + "wss://%s%s/agents/%s/endpoint/protocols/voice?api-version=%s", + u.Host, strings.TrimRight(u.Path, "/"), agentName, agent_api.AgentEndpointAPIVersion, + ) +} + // ProtocolEnvSuffix pairs a user-facing label with the env var suffix // used in AGENT_{KEY}_{SUFFIX}_ENDPOINT variables. type ProtocolEnvSuffix struct { @@ -2238,11 +2251,11 @@ func (p *AgentServiceTargetProvider) deployVoiceAgent( agentClient := agent_api.NewAgentClient(projectEndpoint, p.credential) serviceKey := p.getServiceKey(serviceConfig.Name) - agentObject, err := p.deployVoiceAgentWithMode( - ctx, agentClient, request, apiMode, serviceKey, azdEnv, progress, + agentObject, deployOp, err := p.deployVoiceAgentWithMode( + ctx, agentClient, request, apiMode, azdEnv, progress, ) if err != nil { - return nil, exterrors.ServiceFromAzure(err, exterrors.OpCreateAgent) + return nil, exterrors.ServiceFromAzure(err, deployOp) } fmt.Fprintf(os.Stderr, "Voice agent '%s' deployed successfully!\n", agentObject.Name) @@ -2250,8 +2263,14 @@ func (p *AgentServiceTargetProvider) deployVoiceAgent( // Persist NAME first and ENDPOINT last. ENDPOINT is used as the voice deploy // completion marker by other commands, so avoid writing it before NAME. baseEndpoint := voiceAgentEndpoint(projectEndpoint, agentObject.Name, apiMode) + versionKey := fmt.Sprintf("AGENT_%s_VERSION", serviceKey) + versionValue := "" + if apiMode != voiceAgentAPIModeLegacy { + versionValue = agentObject.Versions.Latest.Version + } for _, envVar := range []struct{ key, value string }{ {fmt.Sprintf("AGENT_%s_NAME", serviceKey), agentObject.Name}, + {versionKey, versionValue}, {fmt.Sprintf("AGENT_%s_ENDPOINT", serviceKey), baseEndpoint}, } { if _, setErr := p.azdClient.Environment().SetValue(ctx, &azdext.SetEnvRequest{ @@ -2282,28 +2301,39 @@ func (p *AgentServiceTargetProvider) deployVoiceAgentWithMode( agentClient *agent_api.AgentClient, request *agent_api.CreateAgentRequest, apiMode voiceAgentAPIMode, - serviceKey string, azdEnv map[string]string, progress azdext.ProgressReporter, -) (*agent_api.AgentObject, error) { +) (*agent_api.AgentObject, string, error) { overriddenHost := azdEnv[voiceOverriddenHostEnvKey] if apiMode == voiceAgentAPIModeLegacy { progress("Creating voice agent using legacy API") - return agentClient.CreateVoiceAgent(ctx, request, agent_api.AgentEndpointAPIVersion, overriddenHost) + agentObject, err := agentClient.CreateVoiceAgent(ctx, request, agent_api.AgentEndpointAPIVersion, overriddenHost) + return agentObject, exterrors.OpCreateAgent, err } - if existingName := strings.TrimSpace(azdEnv[fmt.Sprintf("AGENT_%s_NAME", serviceKey)]); existingName != "" { + remoteAgent, getErr := agentClient.GetVoiceAgentUnified( + ctx, request.Name, agent_api.AgentEndpointAPIVersion, overriddenHost, + ) + if getErr == nil && remoteAgent != nil { progress("Updating voice agent using unified API") updateRequest := &agent_api.UpdateAgentRequest{ CreateAgentVersionRequest: request.CreateAgentVersionRequest, } - return agentClient.UpdateVoiceAgentUnified( - ctx, existingName, updateRequest, agent_api.AgentEndpointAPIVersion, overriddenHost, + agentObject, err := agentClient.UpdateVoiceAgentUnified( + ctx, request.Name, updateRequest, agent_api.AgentEndpointAPIVersion, overriddenHost, ) + return agentObject, exterrors.OpUpdateAgent, err + } + if getErr != nil { + var respErr *azcore.ResponseError + if !errors.As(getErr, &respErr) || respErr.StatusCode != http.StatusNotFound { + return nil, exterrors.OpCreateAgent, getErr + } } progress("Creating voice agent using unified API") - return agentClient.CreateVoiceAgentUnified(ctx, request, agent_api.AgentEndpointAPIVersion, overriddenHost) + agentObject, err := agentClient.CreateVoiceAgentUnified(ctx, request, agent_api.AgentEndpointAPIVersion, overriddenHost) + return agentObject, exterrors.OpCreateAgent, err } func voiceAgentEndpoint(projectEndpoint string, agentName string, apiMode voiceAgentAPIMode) string { @@ -2311,7 +2341,7 @@ func voiceAgentEndpoint(projectEndpoint string, agentName string, apiMode voiceA if apiMode == voiceAgentAPIModeLegacy { return fmt.Sprintf("%s/voice_agents/%s", trimmedEndpoint, agentName) } - return fmt.Sprintf("%s/agents/%s/endpoint/protocols/voice", trimmedEndpoint, agentName) + return buildVoiceWSProtocolURL(trimmedEndpoint, agentName) } // packageCodeDeploy creates a ZIP archive of the agent source code, writes it to a temp file, diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go index 4f2f230dafc..c6e339cd615 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go @@ -701,11 +701,23 @@ func TestVoiceAgentEndpoint_ByMode(t *testing.T) { ) require.Equal( t, - "https://proj.services.ai.azure.com/api/projects/p/agents/my-agent/endpoint/protocols/voice", + "wss://proj.services.ai.azure.com/api/projects/p/agents/my-agent/endpoint/protocols/voice?api-version=v1", voiceAgentEndpoint(projectEndpoint, "my-agent", voiceAgentAPIModeUnified), ) } +func TestBuildVoiceWSProtocolURL(t *testing.T) { + got := buildVoiceWSProtocolURL( + "https://acct.services.ai.azure.com/api/projects/proj/", + "voice-agent", + ) + require.Equal( + t, + "wss://acct.services.ai.azure.com/api/projects/proj/agents/voice-agent/endpoint/protocols/voice?api-version=v1", + got, + ) +} + func createSymlinkOrSkip(t *testing.T, oldname, newname string) { t.Helper() From e83f950447b6b083ca591b7c438166125938bcc7 Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Thu, 20 Aug 2026 10:58:08 +0800 Subject: [PATCH 04/38] fix(agents): validate unified voice response --- .../internal/project/service_target_agent.go | 16 ++++++++++ .../project/service_target_agent_test.go | 30 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go index 6c0c15c8d15..128fa907d7f 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go @@ -2257,6 +2257,9 @@ func (p *AgentServiceTargetProvider) deployVoiceAgent( if err != nil { return nil, exterrors.ServiceFromAzure(err, deployOp) } + if err := validateVoiceAgentDeployResponse(agentObject, apiMode); err != nil { + return nil, err + } fmt.Fprintf(os.Stderr, "Voice agent '%s' deployed successfully!\n", agentObject.Name) @@ -2296,6 +2299,19 @@ func (p *AgentServiceTargetProvider) deployVoiceAgent( return &azdext.ServiceDeployResult{Artifacts: artifacts}, nil } +func validateVoiceAgentDeployResponse(agentObject *agent_api.AgentObject, apiMode voiceAgentAPIMode) error { + if agentObject == nil { + return fmt.Errorf("malformed voice agent service response: missing agent object") + } + if strings.TrimSpace(agentObject.Name) == "" { + return fmt.Errorf("malformed voice agent service response: missing agent name") + } + if apiMode != voiceAgentAPIModeLegacy && strings.TrimSpace(agentObject.Versions.Latest.Version) == "" { + return fmt.Errorf("malformed voice agent service response: missing latest agent version") + } + return nil +} + func (p *AgentServiceTargetProvider) deployVoiceAgentWithMode( ctx context.Context, agentClient *agent_api.AgentClient, diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go index c6e339cd615..6c3fa573ed6 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go @@ -718,6 +718,36 @@ func TestBuildVoiceWSProtocolURL(t *testing.T) { ) } +func TestValidateVoiceAgentDeployResponse(t *testing.T) { + t.Run("legacy requires name only", func(t *testing.T) { + err := validateVoiceAgentDeployResponse( + &agent_api.AgentObject{Name: "voice-agent"}, + voiceAgentAPIModeLegacy, + ) + require.NoError(t, err) + }) + + t.Run("unified requires latest version", func(t *testing.T) { + agent := &agent_api.AgentObject{Name: "voice-agent"} + agent.Versions.Latest.Version = "1" + err := validateVoiceAgentDeployResponse(agent, voiceAgentAPIModeUnifiedFlat) + require.NoError(t, err) + }) + + t.Run("missing name rejected", func(t *testing.T) { + err := validateVoiceAgentDeployResponse(&agent_api.AgentObject{}, voiceAgentAPIModeLegacy) + require.ErrorContains(t, err, "missing agent name") + }) + + t.Run("unified missing version rejected", func(t *testing.T) { + err := validateVoiceAgentDeployResponse( + &agent_api.AgentObject{Name: "voice-agent"}, + voiceAgentAPIModeUnified, + ) + require.ErrorContains(t, err, "missing latest agent version") + }) +} + func createSymlinkOrSkip(t *testing.T, oldname, newname string) { t.Helper() From 048ff7157ae2af1202b81e570658d12c6ca6e312 Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Thu, 20 Aug 2026 12:13:37 +0800 Subject: [PATCH 05/38] fix(agents): address unified voice review feedback --- .../internal/cmd/nextstep/state.go | 27 ++++++------ .../internal/cmd/nextstep/state_test.go | 23 +++++++++-- .../pkg/agents/agent_yaml/map_voice_test.go | 41 +++++++++++++++++++ .../internal/project/service_target_agent.go | 5 +-- 4 files changed, 75 insertions(+), 21 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/nextstep/state.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/nextstep/state.go index 5ffe129eeb1..afafade125c 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/nextstep/state.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/nextstep/state.go @@ -822,23 +822,22 @@ func isDeployed( *errs = append(*errs, fmt.Errorf("read %s: %w", key, err)) return false } - if value != "" { - return true - } - - // Voice agents (kind: prompt-voice) deploy without an agent-version object, - // so they never set AGENT__VERSION. Fall back to the base endpoint - // marker, which every voice deploy writes, so a successfully created voice - // agent is not reported as undeployed. Gate this on the service's actual - // declared kind: a hosted agent whose deploy partially failed can also - // present an empty VERSION with a lingering ENDPOINT, and must stay reported - // as not-deployed. This mirrors the kind gate in + if !isVoice { + return value != "" + } + + // Voice deploys use the base ENDPOINT env var as the completion marker. The + // legacy voice API does not produce AGENT__VERSION, and unified voice + // deploys write VERSION before ENDPOINT to keep ENDPOINT as the final marker. + // Require ENDPOINT for voice even when VERSION is present, otherwise a partial + // env write could be reported as deployed before the callable endpoint was + // persisted. Gate this on the service's actual declared kind: a hosted agent + // whose deploy partially failed can also present an empty VERSION with a + // lingering ENDPOINT, and must stay reported as not-deployed. This mirrors the + // kind gate in // AgentServiceTargetProvider.Endpoints (project package); the two live in // separate packages because project imports nextstep, so a literally shared // helper would create an import cycle. - if !isVoice { - return false - } endpointKey := fmt.Sprintf(agentEndpointVarFormat, serviceKey(serviceName)) endpointValue, err := src.EnvValue(ctx, envName, endpointKey) if err != nil { diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/nextstep/state_test.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/nextstep/state_test.go index c62408b10bf..7f50fe6f190 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/nextstep/state_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/nextstep/state_test.go @@ -739,10 +739,10 @@ func TestServiceKey(t *testing.T) { } } -// TestIsDeployed_VoiceEndpointFallback verifies that a voice agent — which sets -// only AGENT__NAME and AGENT__ENDPOINT, never AGENT__VERSION — is -// still reported as deployed via the base endpoint marker, while an agent with -// neither version nor endpoint is reported undeployed. +// TestIsDeployed_VoiceEndpointFallback verifies that voice readiness is based +// on the base endpoint marker. Legacy voice agents never set VERSION, while +// unified voice agents set VERSION before ENDPOINT; in both cases ENDPOINT is +// the deploy completion marker. func TestIsDeployed_VoiceEndpointFallback(t *testing.T) { t.Parallel() @@ -757,6 +757,21 @@ func TestIsDeployed_VoiceEndpointFallback(t *testing.T) { values: map[string]string{"env1/AGENT_VOICE_SVC_VERSION": "1"}, want: true, }, + { + name: "version set but endpoint missing: undeployed (voice agent partial write)", + values: map[string]string{"env1/AGENT_VOICE_SVC_VERSION": "1"}, + isVoice: true, + want: false, + }, + { + name: "version and endpoint set: deployed (unified voice agent)", + values: map[string]string{ + "env1/AGENT_VOICE_SVC_VERSION": "1", + "env1/AGENT_VOICE_SVC_ENDPOINT": "wss://x/agents/a/endpoint/protocols/voice?api-version=v1", + }, + isVoice: true, + want: true, + }, { name: "no version but base endpoint set: deployed (voice agent)", values: map[string]string{"env1/AGENT_VOICE_SVC_ENDPOINT": "https://x/voice_agents/a"}, diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go index 2ce594e1fbf..1e536b344da 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go @@ -4,6 +4,7 @@ package agent_yaml import ( + "encoding/json" "testing" "azureaiagent/internal/pkg/agents/agent_api" @@ -222,6 +223,46 @@ func TestCreateVoiceAgentAPIRequestFlat_AzureVoiceLocale(t *testing.T) { } } +func TestCreateVoiceAgentAPIRequestFlat_MarshalWireShape(t *testing.T) { + t.Parallel() + voice := "en-US-Ava:DragonHDLatestNeural" + agent := VoiceAgent{ + AgentDefinition: AgentDefinition{Kind: AgentKindPromptVoice, Name: "voice-flat"}, + Model: &Model{Id: "gpt-realtime"}, + Voice: &voice, + } + + req, err := CreateVoiceAgentAPIRequestFlat(agent) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + payload, err := json.Marshal(req) + if err != nil { + t.Fatalf("marshal request: %v", err) + } + + var wire map[string]any + if err := json.Unmarshal(payload, &wire); err != nil { + t.Fatalf("unmarshal payload: %v", err) + } + definition := wire["definition"].(map[string]any) + audio := definition["audio"].(map[string]any) + output := audio["output"].(map[string]any) + + if got, ok := output["voice"].(string); !ok || got != voice { + t.Fatalf("audio.output.voice = %#v, want string %q", output["voice"], voice) + } + if got := output["voice_type"]; got != "azure-standard" { + t.Fatalf("audio.output.voice_type = %#v, want azure-standard", got) + } + if got := output["voice_locale"]; got != "en-US" { + t.Fatalf("audio.output.voice_locale = %#v, want en-US", got) + } + if _, exists := output["type"]; exists { + t.Fatalf("audio.output.type should not be present in flat wire shape: %#v", output) + } +} + // TestCreateVoiceAgentAPIRequest_ExplicitManaged verifies that explicitly // setting model_type: managed is accepted (idempotent with the default). func TestCreateVoiceAgentAPIRequest_ExplicitManaged(t *testing.T) { diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go index 128fa907d7f..a4558598fcb 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go @@ -2218,7 +2218,7 @@ func (p *AgentServiceTargetProvider) deployVoiceAgent( apiMode, err := resolveVoiceAgentAPIMode(azdEnv) if err != nil { return nil, exterrors.Validation( - exterrors.CodeInvalidAgentManifest, + exterrors.CodeInvalidParameter, err.Error(), fmt.Sprintf("set %s to legacy, unified, or unified-flat", voiceAgentAPIEnvKey), ) @@ -2341,8 +2341,7 @@ func (p *AgentServiceTargetProvider) deployVoiceAgentWithMode( return agentObject, exterrors.OpUpdateAgent, err } if getErr != nil { - var respErr *azcore.ResponseError - if !errors.As(getErr, &respErr) || respErr.StatusCode != http.StatusNotFound { + if respErr, ok := errors.AsType[*azcore.ResponseError](getErr); !ok || respErr.StatusCode != http.StatusNotFound { return nil, exterrors.OpCreateAgent, getErr } } From f74dc273a3af77aeacbacbb57119fb90b1a17b0c Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Thu, 20 Aug 2026 14:14:47 +0800 Subject: [PATCH 06/38] feat(agents): support advanced voice settings --- cli/azd/extensions/azure.ai.agents/README.md | 92 ++++++++ .../extensions/azure.ai.agents/cspell.yaml | 3 + .../internal/pkg/agents/agent_api/models.go | 83 +++++-- .../internal/pkg/agents/agent_yaml/map.go | 216 ++++++++++++++++-- .../pkg/agents/agent_yaml/map_voice_test.go | 102 ++++++++- .../internal/pkg/agents/agent_yaml/parse.go | 69 ++++++ .../internal/pkg/agents/agent_yaml/yaml.go | 89 ++++++++ .../internal/project/agent_definition.go | 67 ++++-- .../internal/project/service_target_agent.go | 21 ++ .../project/service_target_agent_test.go | 8 + .../schemas/azure.ai.agent.json | 159 ++++++++++++- 11 files changed, 836 insertions(+), 73 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/README.md b/cli/azd/extensions/azure.ai.agents/README.md index 6fecb7e17db..e32553da8d9 100644 --- a/cli/azd/extensions/azure.ai.agents/README.md +++ b/cli/azd/extensions/azure.ai.agents/README.md @@ -164,6 +164,98 @@ Details: still require `legacy` or `unified` until the flat output shape is fully rolled out. +## Prompt voice advanced configuration + +Advanced prompt voice settings are authored on the `azure.ai.agent` service in +`azure.yaml` and require the unified flat API mode: + +```bash +azd env set AZURE_VOICE_AGENT_API unified-flat +``` + +```yaml +services: + voice-agent: + host: azure.ai.agent + kind: prompt-voice + name: voice-agent + modelType: managed # or self_deployed for BYOM + model: + id: gpt-realtime + instructions: You are {{persona}}, a concise voice assistant. + structuredInputs: + persona: + description: Assistant persona + defaultValue: Ada + schema: + type: string + audio: + input: + format: + type: audio/pcmu + noiseReduction: + type: near_field + echoCancellation: + type: server_echo_cancellation + reference_source: server + channels: 1 + turnDetection: + type: azure_semantic_vad + threshold: 0.6 + speechDurationMs: 120 + removeFillerWords: true + createResponse: true + interruptResponse: true + languages: [en-US] + autoTruncate: true + transcription: + model: whisper-1 + language: en-US + output: + format: + type: audio/pcm + rate: 24000 + voice: + type: azure_standard + name: en-US-AvaNeural + locale: en-US + style: cheerful + speed: 1.0 + outputModalities: [audio, text] + tools: + - type: system + name: end_conversation + avatar: + type: video_avatar + character: lisa + style: casual-sitting + output_protocol: webrtc + greeting: + type: template + text: Hello {{persona}} + toolChoice: auto + parallelToolCalls: true + maxOutputTokens: inf + include: + - item.input_audio_transcription.phrases +``` + +Notes: + +- `voice`, `instructions`, and `store` remain supported for simple prompt voice + agents. Missing audio fields keep the existing azd defaults. +- `audio.output.voice` uses an author-friendly object shape; in `unified-flat` + mode azd maps it to the service flat fields `voice`, `voice_type`, + `voice_locale`, `style`, `pitch`, `rate`, and `volume`. +- `structuredInputs.defaultValue` maps to the service wire field + `default_value`. +- Direct voice tool types are `function`, `mcp`, `system`, and `toolbox`. + Server-side tools such as `web_search`, `azure_ai_search`, and `openapi` must + be packaged through a toolbox. +- `tools`, `avatar`, `greeting`, `handoff`, `toolChoice`, and + `echoCancellation` intentionally remain light pass-through blocks so azd does + not block new service-side additions. + ## Session idle timeout A hosted agent's runtime session sandbox is suspended by Foundry after a period diff --git a/cli/azd/extensions/azure.ai.agents/cspell.yaml b/cli/azd/extensions/azure.ai.agents/cspell.yaml index 5d2d9e0950c..78be91d2fea 100644 --- a/cli/azd/extensions/azure.ai.agents/cspell.yaml +++ b/cli/azd/extensions/azure.ai.agents/cspell.yaml @@ -11,6 +11,9 @@ words: # Voice (prompt-voice) agents - BYOM - Nanami + - pcma + - pcmu + - webrtc # Azure region names - australiaeast - brazilsouth diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go index 96b33806e95..b278ab2ec64 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go @@ -312,7 +312,7 @@ const ( // VoiceAudioFormat describes a PCM audio stream format (e.g. audio/pcm @ 24 kHz). type VoiceAudioFormat struct { Type string `json:"type"` - Rate int `json:"rate"` + Rate *int `json:"rate,omitempty"` } // VoiceTurnDetection configures server-side voice-activity detection so the @@ -322,32 +322,54 @@ type VoiceTurnDetection struct { Threshold *float64 `json:"threshold,omitempty"` PrefixPaddingMs *int `json:"prefix_padding_ms,omitempty"` SilenceDurationMs *int `json:"silence_duration_ms,omitempty"` + CreateResponse *bool `json:"create_response,omitempty"` + Eagerness *string `json:"eagerness,omitempty"` + SpeechDurationMs *int `json:"speech_duration_ms,omitempty"` + RemoveFillerWords *bool `json:"remove_filler_words,omitempty"` + InterruptResponse *bool `json:"interrupt_response,omitempty"` + Languages []string `json:"languages,omitempty"` + AutoTruncate *bool `json:"auto_truncate,omitempty"` } // VoiceTranscription enables user-speech transcription events on the input stream. type VoiceTranscription struct { - Model string `json:"model,omitempty"` + Model string `json:"model,omitempty"` + Language *string `json:"language,omitempty"` + Prompt *string `json:"prompt,omitempty"` +} + +// VoiceNoiseReduction configures input audio noise reduction. +type VoiceNoiseReduction struct { + Type string `json:"type"` } // VoiceInputConfig is the input (caller -> agent) audio configuration. type VoiceInputConfig struct { - Format *VoiceAudioFormat `json:"format,omitempty"` - TurnDetection *VoiceTurnDetection `json:"turn_detection,omitempty"` - Transcription *VoiceTranscription `json:"transcription,omitempty"` + Format *VoiceAudioFormat `json:"format,omitempty"` + NoiseReduction *VoiceNoiseReduction `json:"noise_reduction,omitempty"` + EchoCancellation map[string]any `json:"echo_cancellation,omitempty"` + TurnDetection *VoiceTurnDetection `json:"turn_detection,omitempty"` + Transcription *VoiceTranscription `json:"transcription,omitempty"` } // VoiceConfig selects the output voice. Type is "openai" for realtime voices // (single lowercase word, e.g. "alloy") or "azure_standard" for Azure Neural // voices (e.g. "en-US-Ava:DragonHDLatestNeural"). type VoiceConfig struct { - Type string `json:"type"` - Name string `json:"name"` + Type string `json:"type"` + Name string `json:"name"` + Style *string `json:"style,omitempty"` + Pitch *string `json:"pitch,omitempty"` + Rate *string `json:"rate,omitempty"` + Locale *string `json:"locale,omitempty"` + Volume *string `json:"volume,omitempty"` } // VoiceOutputConfig is the output (agent -> caller) audio configuration. type VoiceOutputConfig struct { Format *VoiceAudioFormat `json:"format,omitempty"` Voice *VoiceConfig `json:"voice,omitempty"` + Speed *float64 `json:"speed,omitempty"` } // VoiceOutputConfigFlat is the newer Voice Live output shape used by the @@ -358,6 +380,11 @@ type VoiceOutputConfigFlat struct { Voice string `json:"voice,omitempty"` VoiceType string `json:"voice_type,omitempty"` VoiceLocale string `json:"voice_locale,omitempty"` + Style *string `json:"style,omitempty"` + Pitch *string `json:"pitch,omitempty"` + Rate *string `json:"rate,omitempty"` + Volume *string `json:"volume,omitempty"` + Speed *float64 `json:"speed,omitempty"` } // VoiceAudioConfig bundles the input and output audio configuration. @@ -377,12 +404,21 @@ type VoiceAudioConfigFlat struct { // is always AgentKindVoice ("voice"). type VoiceAgentDefinition struct { AgentDefinition - ModelType VoiceModelType `json:"model_type"` - Model string `json:"model"` - Instructions string `json:"instructions,omitempty"` - Audio *VoiceAudioConfig `json:"audio,omitempty"` - OutputModalities []string `json:"output_modalities,omitempty"` - Store *bool `json:"store,omitempty"` + ModelType VoiceModelType `json:"model_type"` + Model string `json:"model"` + Instructions string `json:"instructions,omitempty"` + StructuredInputs map[string]any `json:"structured_inputs,omitempty"` + Audio *VoiceAudioConfig `json:"audio,omitempty"` + OutputModalities []string `json:"output_modalities,omitempty"` + Store *bool `json:"store,omitempty"` + Tools []map[string]any `json:"tools,omitempty"` + Avatar map[string]any `json:"avatar,omitempty"` + Greeting map[string]any `json:"greeting,omitempty"` + Handoff map[string]any `json:"handoff,omitempty"` + ToolChoice any `json:"tool_choice,omitempty"` + ParallelToolCalls *bool `json:"parallel_tool_calls,omitempty"` + MaxOutputTokens any `json:"max_output_tokens,omitempty"` + Include []string `json:"include,omitempty"` } // VoiceAgentDefinitionFlat is the voice definition shape aligned with the @@ -390,12 +426,21 @@ type VoiceAgentDefinition struct { // provider details are sibling fields. type VoiceAgentDefinitionFlat struct { AgentDefinition - ModelType VoiceModelType `json:"model_type"` - Model string `json:"model"` - Instructions string `json:"instructions,omitempty"` - Audio *VoiceAudioConfigFlat `json:"audio,omitempty"` - OutputModalities []string `json:"output_modalities,omitempty"` - Store *bool `json:"store,omitempty"` + ModelType VoiceModelType `json:"model_type"` + Model string `json:"model"` + Instructions string `json:"instructions,omitempty"` + StructuredInputs map[string]any `json:"structured_inputs,omitempty"` + Audio *VoiceAudioConfigFlat `json:"audio,omitempty"` + OutputModalities []string `json:"output_modalities,omitempty"` + Store *bool `json:"store,omitempty"` + Tools []map[string]any `json:"tools,omitempty"` + Avatar map[string]any `json:"avatar,omitempty"` + Greeting map[string]any `json:"greeting,omitempty"` + Handoff map[string]any `json:"handoff,omitempty"` + ToolChoice any `json:"tool_choice,omitempty"` + ParallelToolCalls *bool `json:"parallel_tool_calls,omitempty"` + MaxOutputTokens any `json:"max_output_tokens,omitempty"` + Include []string `json:"include,omitempty"` } // CreateAgentVersionRequest represents a request to create an agent version diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go index 0404762cb43..f017c4875de 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go @@ -563,10 +563,22 @@ func flatVoiceType(voice *agent_api.VoiceConfig) string { return voice.Type } +func normalizeFlatVoiceType(voiceType string) string { + switch strings.TrimSpace(voiceType) { + case "azure_standard": + return "azure-standard" + default: + return strings.TrimSpace(voiceType) + } +} + func flatVoiceLocale(voice *agent_api.VoiceConfig) string { if voice == nil || voice.Name == "" || isOpenAIVoice(voice.Name) { return "" } + if voice.Locale != nil && strings.TrimSpace(*voice.Locale) != "" { + return strings.TrimSpace(*voice.Locale) + } parts := strings.SplitN(voice.Name, "-", 3) if len(parts) < 2 { return "" @@ -574,6 +586,114 @@ func flatVoiceLocale(voice *agent_api.VoiceConfig) string { return parts[0] + "-" + parts[1] } +func defaultVoiceAudioFormat() *agent_api.VoiceAudioFormat { + rate := defaultVoiceAudioRate + return &agent_api.VoiceAudioFormat{Type: defaultVoiceAudioType, Rate: &rate} +} + +func mapVoiceAudioFormat(format *VoiceAudioFormat, fallback *agent_api.VoiceAudioFormat) *agent_api.VoiceAudioFormat { + out := &agent_api.VoiceAudioFormat{} + if fallback != nil { + *out = *fallback + } + if format != nil { + if strings.TrimSpace(format.Type) != "" { + out.Type = strings.TrimSpace(format.Type) + } + if format.Rate != nil { + out.Rate = format.Rate + } + } + return out +} + +func mapVoiceTurnDetection(turnDetection *VoiceTurnDetection) *agent_api.VoiceTurnDetection { + out := &agent_api.VoiceTurnDetection{Type: defaultVoiceTurnDetectionType} + if turnDetection == nil { + return out + } + if strings.TrimSpace(turnDetection.Type) != "" { + out.Type = strings.TrimSpace(turnDetection.Type) + } + out.Threshold = turnDetection.Threshold + out.PrefixPaddingMs = turnDetection.PrefixPaddingMs + out.SilenceDurationMs = turnDetection.SilenceDurationMs + out.CreateResponse = turnDetection.CreateResponse + out.Eagerness = turnDetection.Eagerness + out.SpeechDurationMs = turnDetection.SpeechDurationMs + out.RemoveFillerWords = turnDetection.RemoveFillerWords + out.InterruptResponse = turnDetection.InterruptResponse + out.Languages = turnDetection.Languages + out.AutoTruncate = turnDetection.AutoTruncate + return out +} + +func mapVoiceTranscription(transcription *VoiceTranscription) *agent_api.VoiceTranscription { + out := &agent_api.VoiceTranscription{Model: defaultVoiceInputTranscriptionModel} + if transcription == nil { + return out + } + if strings.TrimSpace(transcription.Model) != "" { + out.Model = strings.TrimSpace(transcription.Model) + } + out.Language = transcription.Language + out.Prompt = transcription.Prompt + return out +} + +func mapVoiceConfig(voice *VoiceConfig, fallbackName string) *agent_api.VoiceConfig { + if voice == nil { + return buildVoiceConfig(fallbackName) + } + name := strings.TrimSpace(voice.Name) + if name == "" { + name = fallbackName + } + voiceType := strings.TrimSpace(voice.Type) + if voiceType == "" { + out := buildVoiceConfig(name) + out.Style = voice.Style + out.Pitch = voice.Pitch + out.Rate = voice.Rate + out.Locale = voice.Locale + out.Volume = voice.Volume + return out + } + return &agent_api.VoiceConfig{ + Type: voiceType, + Name: name, + Style: voice.Style, + Pitch: voice.Pitch, + Rate: voice.Rate, + Locale: voice.Locale, + Volume: voice.Volume, + } +} + +func mapVoiceStructuredInputs(inputs map[string]any) map[string]any { + if len(inputs) == 0 { + return nil + } + out := make(map[string]any, len(inputs)) + for name, input := range inputs { + inputMap, ok := input.(map[string]any) + if !ok { + out[name] = input + continue + } + + mapped := maps.Clone(inputMap) + if value, ok := mapped["defaultValue"]; ok { + if _, hasSnakeCase := mapped["default_value"]; !hasSnakeCase { + mapped["default_value"] = value + } + delete(mapped, "defaultValue") + } + out[name] = mapped + } + return out +} + // CreateVoiceAgentAPIRequest builds a CreateAgentRequest for a declarative // voice agent. It translates the authoring kind "prompt-voice" into the // data-plane service kind "voice" and defaults the audio pipeline. @@ -616,37 +736,77 @@ func createVoiceAgentAPIRequest(voiceAgent VoiceAgent, flatOutput bool) (*agent_ voiceName = *voiceAgent.Voice } - audioFormat := &agent_api.VoiceAudioFormat{ - Type: defaultVoiceAudioType, - Rate: defaultVoiceAudioRate, + inputFormat := defaultVoiceAudioFormat() + outputFormat := defaultVoiceAudioFormat() + turnDetection := mapVoiceTurnDetection(nil) + transcription := mapVoiceTranscription(nil) + var noiseReduction *agent_api.VoiceNoiseReduction + var echoCancellation map[string]any + outputVoice := buildVoiceConfig(voiceName) + var outputSpeed *float64 + if voiceAgent.Audio != nil { + if voiceAgent.Audio.Input != nil { + inputFormat = mapVoiceAudioFormat(voiceAgent.Audio.Input.Format, inputFormat) + if voiceAgent.Audio.Input.NoiseReduction != nil { + noiseReduction = &agent_api.VoiceNoiseReduction{Type: strings.TrimSpace(voiceAgent.Audio.Input.NoiseReduction.Type)} + } + echoCancellation = voiceAgent.Audio.Input.EchoCancellation + turnDetection = mapVoiceTurnDetection(voiceAgent.Audio.Input.TurnDetection) + transcription = mapVoiceTranscription(voiceAgent.Audio.Input.Transcription) + } + if voiceAgent.Audio.Output != nil { + outputFormat = mapVoiceAudioFormat(voiceAgent.Audio.Output.Format, outputFormat) + outputVoice = mapVoiceConfig(voiceAgent.Audio.Output.Voice, voiceName) + outputSpeed = voiceAgent.Audio.Output.Speed + } + } + + outputModalities := []string{"audio"} + if len(voiceAgent.OutputModalities) > 0 { + outputModalities = voiceAgent.OutputModalities } input := &agent_api.VoiceInputConfig{ - Format: audioFormat, - TurnDetection: &agent_api.VoiceTurnDetection{Type: defaultVoiceTurnDetectionType}, - Transcription: &agent_api.VoiceTranscription{Model: defaultVoiceInputTranscriptionModel}, + Format: inputFormat, + NoiseReduction: noiseReduction, + EchoCancellation: echoCancellation, + TurnDetection: turnDetection, + Transcription: transcription, } - voiceConfig := buildVoiceConfig(voiceName) if flatOutput { voiceDef := agent_api.VoiceAgentDefinitionFlat{ AgentDefinition: agent_api.AgentDefinition{ // Translate authoring kind prompt-voice -> service kind voice. Kind: agent_api.AgentKindVoice, }, - ModelType: modelType, - Model: modelID, - Instructions: instructions, + ModelType: modelType, + Model: modelID, + Instructions: instructions, + StructuredInputs: mapVoiceStructuredInputs(voiceAgent.StructuredInputs), Audio: &agent_api.VoiceAudioConfigFlat{ Input: input, Output: &agent_api.VoiceOutputConfigFlat{ - Format: audioFormat, - Voice: voiceConfig.Name, - VoiceType: flatVoiceType(voiceConfig), - VoiceLocale: flatVoiceLocale(voiceConfig), + Format: outputFormat, + Voice: outputVoice.Name, + VoiceType: normalizeFlatVoiceType(flatVoiceType(outputVoice)), + VoiceLocale: flatVoiceLocale(outputVoice), + Style: outputVoice.Style, + Pitch: outputVoice.Pitch, + Rate: outputVoice.Rate, + Volume: outputVoice.Volume, + Speed: outputSpeed, }, }, - OutputModalities: []string{"audio"}, - Store: voiceAgent.Store, + OutputModalities: outputModalities, + Store: voiceAgent.Store, + Tools: voiceAgent.Tools, + Avatar: voiceAgent.Avatar, + Greeting: voiceAgent.Greeting, + Handoff: voiceAgent.Handoff, + ToolChoice: voiceAgent.ToolChoice, + ParallelToolCalls: voiceAgent.ParallelToolCalls, + MaxOutputTokens: voiceAgent.MaxOutputTokens, + Include: voiceAgent.Include, } return createAgentAPIRequest(voiceAgent.AgentDefinition, voiceDef, nil, nil) @@ -657,18 +817,28 @@ func createVoiceAgentAPIRequest(voiceAgent VoiceAgent, flatOutput bool) (*agent_ // Translate authoring kind prompt-voice -> service kind voice. Kind: agent_api.AgentKindVoice, }, - ModelType: modelType, - Model: modelID, - Instructions: instructions, + ModelType: modelType, + Model: modelID, + Instructions: instructions, + StructuredInputs: mapVoiceStructuredInputs(voiceAgent.StructuredInputs), Audio: &agent_api.VoiceAudioConfig{ Input: input, Output: &agent_api.VoiceOutputConfig{ - Format: audioFormat, - Voice: voiceConfig, + Format: outputFormat, + Voice: outputVoice, + Speed: outputSpeed, }, }, - OutputModalities: []string{"audio"}, - Store: voiceAgent.Store, + OutputModalities: outputModalities, + Store: voiceAgent.Store, + Tools: voiceAgent.Tools, + Avatar: voiceAgent.Avatar, + Greeting: voiceAgent.Greeting, + Handoff: voiceAgent.Handoff, + ToolChoice: voiceAgent.ToolChoice, + ParallelToolCalls: voiceAgent.ParallelToolCalls, + MaxOutputTokens: voiceAgent.MaxOutputTokens, + Include: voiceAgent.Include, } return createAgentAPIRequest(voiceAgent.AgentDefinition, voiceDef, nil, nil) diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go index 1e536b344da..115538c572b 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go @@ -10,6 +10,8 @@ import ( "azureaiagent/internal/pkg/agents/agent_api" ) +func ptr[T any](v T) *T { return &v } + // --------------------------------------------------------------------------- // isOpenAIVoice / buildVoiceConfig // --------------------------------------------------------------------------- @@ -118,7 +120,8 @@ func TestCreateVoiceAgentAPIRequest_Defaults(t *testing.T) { t.Fatalf("Audio pipeline not populated: %+v", def.Audio) } in := def.Audio.Input - if in.Format == nil || in.Format.Type != defaultVoiceAudioType || in.Format.Rate != defaultVoiceAudioRate { + if in.Format == nil || in.Format.Type != defaultVoiceAudioType || in.Format.Rate == nil || + *in.Format.Rate != defaultVoiceAudioRate { t.Errorf("input format = %+v", in.Format) } if in.TurnDetection == nil || in.TurnDetection.Type != defaultVoiceTurnDetectionType { @@ -128,7 +131,8 @@ func TestCreateVoiceAgentAPIRequest_Defaults(t *testing.T) { t.Errorf("transcription = %+v", in.Transcription) } out := def.Audio.Output - if out.Format == nil || out.Format.Type != defaultVoiceAudioType || out.Format.Rate != defaultVoiceAudioRate { + if out.Format == nil || out.Format.Type != defaultVoiceAudioType || out.Format.Rate == nil || + *out.Format.Rate != defaultVoiceAudioRate { t.Errorf("output format = %+v", out.Format) } // Default voice is the DragonHD Azure Neural voice. @@ -263,6 +267,100 @@ func TestCreateVoiceAgentAPIRequestFlat_MarshalWireShape(t *testing.T) { } } +func TestCreateVoiceAgentAPIRequestFlat_AdvancedSettingsWireShape(t *testing.T) { + t.Parallel() + inRate := 16000 + outRate := 24000 + threshold := 0.6 + speechDurationMs := 120 + createResponse := true + removeFillerWords := true + interruptResponse := true + autoTruncate := true + speed := 1.1 + parallelToolCalls := true + style := "cheerful" + pitch := "+0Hz" + rate := "+0%" + volume := "+0%" + agent := VoiceAgent{ + AgentDefinition: AgentDefinition{Kind: AgentKindPromptVoice, Name: "voice-advanced"}, + Model: &Model{Id: "gpt-realtime"}, + Instructions: ptr("You are {{persona}}, a concise voice assistant."), + StructuredInputs: map[string]any{ + "persona": map[string]any{"description": "Assistant persona", "defaultValue": "Ada"}, + }, + Audio: &VoiceAudio{ + Input: &VoiceAudioInput{ + Format: &VoiceAudioFormat{Type: "audio/pcmu", Rate: &inRate}, + NoiseReduction: &VoiceNoiseReduction{Type: "near_field"}, + EchoCancellation: map[string]any{"type": "server_echo_cancellation", "channels": 1}, + TurnDetection: &VoiceTurnDetection{ + Type: "azure_semantic_vad", + Threshold: &threshold, + SpeechDurationMs: &speechDurationMs, + CreateResponse: &createResponse, + RemoveFillerWords: &removeFillerWords, + InterruptResponse: &interruptResponse, + Languages: []string{"en-US"}, + AutoTruncate: &autoTruncate, + }, + Transcription: &VoiceTranscription{Model: "whisper-1", Language: ptr("en-US"), Prompt: ptr("Contoso terms")}, + }, + Output: &VoiceAudioOutput{ + Format: &VoiceAudioFormat{Type: "audio/pcm", Rate: &outRate}, + Voice: &VoiceConfig{ + Type: "azure_standard", Name: "en-US-AvaNeural", Style: &style, + Pitch: &pitch, Rate: &rate, Locale: ptr("en-US"), Volume: &volume, + }, + Speed: &speed, + }, + }, + OutputModalities: []string{"audio", "text"}, + Tools: []map[string]any{{"type": "system", "name": "end_conversation"}}, + Avatar: map[string]any{"type": "video_avatar", "character": "lisa", "output_protocol": "webrtc"}, + Greeting: map[string]any{"type": "template", "text": "Hello {{persona}}"}, + ToolChoice: "auto", + ParallelToolCalls: ¶llelToolCalls, + MaxOutputTokens: "inf", + Include: []string{"item.input_audio_transcription.phrases"}, + } + + req, err := CreateVoiceAgentAPIRequestFlat(agent) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + payload, err := json.Marshal(req) + if err != nil { + t.Fatalf("marshal request: %v", err) + } + + var wire map[string]any + if err := json.Unmarshal(payload, &wire); err != nil { + t.Fatalf("unmarshal payload: %v", err) + } + def := wire["definition"].(map[string]any) + input := def["audio"].(map[string]any)["input"].(map[string]any) + output := def["audio"].(map[string]any)["output"].(map[string]any) + structured := def["structured_inputs"].(map[string]any)["persona"].(map[string]any) + + if structured["default_value"] != "Ada" || structured["defaultValue"] != nil { + t.Fatalf("structured input default was not mapped to wire shape: %#v", structured) + } + if output["voice"] != "en-US-AvaNeural" || output["voice_type"] != "azure-standard" || output["style"] != style { + t.Fatalf("output voice flat shape not mapped: %#v", output) + } + if input["echo_cancellation"].(map[string]any)["type"] != "server_echo_cancellation" { + t.Fatalf("echo cancellation not mapped: %#v", input["echo_cancellation"]) + } + if def["tool_choice"] != "auto" || def["max_output_tokens"] != "inf" { + t.Fatalf("response options not mapped: %#v", def) + } + if len(def["tools"].([]any)) != 1 || def["avatar"].(map[string]any)["character"] != "lisa" { + t.Fatalf("tools/avatar not mapped: %#v", def) + } +} + // TestCreateVoiceAgentAPIRequest_ExplicitManaged verifies that explicitly // setting model_type: managed is accepted (idempotent with the default). func TestCreateVoiceAgentAPIRequest_ExplicitManaged(t *testing.T) { diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go index d61fed8905e..250ca6776d1 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go @@ -440,6 +440,7 @@ func ValidateAgentDefinition(templateBytes []byte) error { "template.model_type '%s' is not supported; use '%s' or '%s'", agent.ModelType, VoiceModelTypeManaged, VoiceModelTypeSelfDeployed)) } + errors = append(errors, validateVoiceAgentAdvancedConfig(agent)...) } else { errors = append(errors, fmt.Sprintf("failed to unmarshal to VoiceAgent: %v", err)) } @@ -459,6 +460,74 @@ func ValidateAgentDefinition(templateBytes []byte) error { return nil } +func validateVoiceAgentAdvancedConfig(agent VoiceAgent) []string { + var errors []string + for i, modality := range agent.OutputModalities { + if strings.TrimSpace(modality) == "" { + errors = append(errors, fmt.Sprintf("template.output_modalities[%d] must not be blank", i)) + } + } + + if agent.Audio == nil { + return errors + } + if agent.Audio.Input != nil { + errors = append(errors, validateVoiceAudioFormat("template.audio.input.format", agent.Audio.Input.Format)...) + if nr := agent.Audio.Input.NoiseReduction; nr != nil && strings.TrimSpace(nr.Type) == "" { + errors = append(errors, "template.audio.input.noise_reduction.type must not be blank") + } + if td := agent.Audio.Input.TurnDetection; td != nil { + if strings.TrimSpace(td.Type) == "" { + errors = append(errors, "template.audio.input.turn_detection.type must not be blank") + } + if td.Threshold != nil && (*td.Threshold < 0 || *td.Threshold > 1) { + errors = append(errors, "template.audio.input.turn_detection.threshold must be between 0 and 1") + } + if td.PrefixPaddingMs != nil && *td.PrefixPaddingMs < 0 { + errors = append(errors, "template.audio.input.turn_detection.prefix_padding_ms must be >= 0") + } + if td.SilenceDurationMs != nil && *td.SilenceDurationMs < 0 { + errors = append(errors, "template.audio.input.turn_detection.silence_duration_ms must be >= 0") + } + if td.SpeechDurationMs != nil && *td.SpeechDurationMs < 0 { + errors = append(errors, "template.audio.input.turn_detection.speech_duration_ms must be >= 0") + } + } + } + if agent.Audio.Output != nil { + errors = append(errors, validateVoiceAudioFormat("template.audio.output.format", agent.Audio.Output.Format)...) + if voice := agent.Audio.Output.Voice; voice != nil { + if strings.TrimSpace(voice.Type) == "" { + errors = append(errors, "template.audio.output.voice.type must not be blank") + } + if strings.TrimSpace(voice.Name) == "" { + errors = append(errors, "template.audio.output.voice.name must not be blank") + } + } + if speed := agent.Audio.Output.Speed; speed != nil && (*speed < 0.25 || *speed > 1.5) { + errors = append(errors, "template.audio.output.speed must be between 0.25 and 1.5") + } + } + return errors +} + +func validateVoiceAudioFormat(path string, format *VoiceAudioFormat) []string { + if format == nil { + return nil + } + var errors []string + formatType := strings.TrimSpace(format.Type) + if formatType == "" { + errors = append(errors, path+".type must not be blank") + } else if formatType != "audio/pcm" && formatType != "audio/pcmu" && formatType != "audio/pcma" { + errors = append(errors, path+".type must be 'audio/pcm', 'audio/pcmu', or 'audio/pcma'") + } + if format.Rate != nil && *format.Rate <= 0 { + errors = append(errors, path+".rate must be greater than 0") + } + return errors +} + // Validate that the agent name matches the expected deployable format func ValidateAgentName(name string) error { if name == "" { diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/yaml.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/yaml.go index fd70eb6aa45..9d55803256e 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/yaml.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/yaml.go @@ -214,9 +214,98 @@ type VoiceAgent struct { // Voice is the output voice name (e.g. "en-US-Ava:DragonHDLatestNeural" for // an Azure Neural voice, or "alloy" for an OpenAI realtime voice). Voice *string `json:"voice,omitempty" yaml:"voice,omitempty"` + // StructuredInputs declares template inputs used by voice instructions and greeting. + StructuredInputs map[string]any `json:"structuredInputs,omitempty" yaml:"structured_inputs,omitempty"` + // Audio customizes the input and output voice pipeline. Missing fields keep azd defaults. + Audio *VoiceAudio `json:"audio,omitempty" yaml:"audio,omitempty"` + // OutputModalities declares response modalities such as audio, text, animation, or avatar. + OutputModalities []string `json:"outputModalities,omitempty" yaml:"output_modalities,omitempty"` // Store toggles server-side logging (transcript + per-turn audio). Optional; // the service defaults to false when omitted. Store *bool `json:"store,omitempty" yaml:"store,omitempty"` + // Tools are passed through to the prompt voice service. Supported direct tool + // types include function, mcp, system, and toolbox. + Tools []map[string]any `json:"tools,omitempty" yaml:"tools,omitempty"` + // Avatar customizes voice avatar output for services that support it. + Avatar map[string]any `json:"avatar,omitempty" yaml:"avatar,omitempty"` + // Greeting configures initial greeting behavior for services that support it. + Greeting map[string]any `json:"greeting,omitempty" yaml:"greeting,omitempty"` + // Handoff configures voice handoff behavior for services that support it. + Handoff map[string]any `json:"handoff,omitempty" yaml:"handoff,omitempty"` + // ToolChoice configures service tool choice behavior, such as auto/none/required. + ToolChoice any `json:"toolChoice,omitempty" yaml:"tool_choice,omitempty"` + // ParallelToolCalls toggles parallel tool calls. + ParallelToolCalls *bool `json:"parallelToolCalls,omitempty" yaml:"parallel_tool_calls,omitempty"` + // MaxOutputTokens limits response output tokens. Use an integer or service-supported string such as "inf". + MaxOutputTokens any `json:"maxOutputTokens,omitempty" yaml:"max_output_tokens,omitempty"` + // Include requests additional service response fields. + Include []string `json:"include,omitempty" yaml:"include,omitempty"` +} + +// VoiceAudio bundles optional prompt voice input/output audio overrides. +type VoiceAudio struct { + Input *VoiceAudioInput `json:"input,omitempty" yaml:"input,omitempty"` + Output *VoiceAudioOutput `json:"output,omitempty" yaml:"output,omitempty"` +} + +// VoiceAudioInput customizes caller-to-agent audio. +type VoiceAudioInput struct { + Format *VoiceAudioFormat `json:"format,omitempty" yaml:"format,omitempty"` + NoiseReduction *VoiceNoiseReduction `json:"noiseReduction,omitempty" yaml:"noise_reduction,omitempty"` + EchoCancellation map[string]any `json:"echoCancellation,omitempty" yaml:"echo_cancellation,omitempty"` + TurnDetection *VoiceTurnDetection `json:"turnDetection,omitempty" yaml:"turn_detection,omitempty"` + Transcription *VoiceTranscription `json:"transcription,omitempty" yaml:"transcription,omitempty"` +} + +// VoiceAudioOutput customizes agent-to-caller audio. +type VoiceAudioOutput struct { + Format *VoiceAudioFormat `json:"format,omitempty" yaml:"format,omitempty"` + Voice *VoiceConfig `json:"voice,omitempty" yaml:"voice,omitempty"` + Speed *float64 `json:"speed,omitempty" yaml:"speed,omitempty"` +} + +// VoiceAudioFormat describes an audio stream format. +type VoiceAudioFormat struct { + Type string `json:"type" yaml:"type"` + Rate *int `json:"rate,omitempty" yaml:"rate,omitempty"` +} + +// VoiceNoiseReduction configures input audio noise reduction. +type VoiceNoiseReduction struct { + Type string `json:"type" yaml:"type"` +} + +// VoiceTurnDetection configures server-side turn detection. +type VoiceTurnDetection struct { + Type string `json:"type" yaml:"type"` + Threshold *float64 `json:"threshold,omitempty" yaml:"threshold,omitempty"` + PrefixPaddingMs *int `json:"prefixPaddingMs,omitempty" yaml:"prefix_padding_ms,omitempty"` + SilenceDurationMs *int `json:"silenceDurationMs,omitempty" yaml:"silence_duration_ms,omitempty"` + CreateResponse *bool `json:"createResponse,omitempty" yaml:"create_response,omitempty"` + Eagerness *string `json:"eagerness,omitempty" yaml:"eagerness,omitempty"` + SpeechDurationMs *int `json:"speechDurationMs,omitempty" yaml:"speech_duration_ms,omitempty"` + RemoveFillerWords *bool `json:"removeFillerWords,omitempty" yaml:"remove_filler_words,omitempty"` + InterruptResponse *bool `json:"interruptResponse,omitempty" yaml:"interrupt_response,omitempty"` + Languages []string `json:"languages,omitempty" yaml:"languages,omitempty"` + AutoTruncate *bool `json:"autoTruncate,omitempty" yaml:"auto_truncate,omitempty"` +} + +// VoiceTranscription configures input transcription. +type VoiceTranscription struct { + Model string `json:"model,omitempty" yaml:"model,omitempty"` + Language *string `json:"language,omitempty" yaml:"language,omitempty"` + Prompt *string `json:"prompt,omitempty" yaml:"prompt,omitempty"` +} + +// VoiceConfig selects the output voice. +type VoiceConfig struct { + Type string `json:"type" yaml:"type"` + Name string `json:"name" yaml:"name"` + Style *string `json:"style,omitempty" yaml:"style,omitempty"` + Pitch *string `json:"pitch,omitempty" yaml:"pitch,omitempty"` + Rate *string `json:"rate,omitempty" yaml:"rate,omitempty"` + Locale *string `json:"locale,omitempty" yaml:"locale,omitempty"` + Volume *string `json:"volume,omitempty" yaml:"volume,omitempty"` } // ContainerResources represents the resource allocation for a containerized agent. diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/agent_definition.go b/cli/azd/extensions/azure.ai.agents/internal/project/agent_definition.go index 26b5a06f874..1625d9a7637 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/agent_definition.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/agent_definition.go @@ -138,35 +138,68 @@ type AgentDefinitionInline struct { // Voice-agent fields (kind: prompt-voice). All omitempty so container/ // workflow entries are byte-for-byte unchanged. - ModelType agent_yaml.VoiceModelType `json:"modelType,omitempty"` - Model *agent_yaml.Model `json:"model,omitempty"` - Instructions *string `json:"instructions,omitempty"` - Voice *string `json:"voice,omitempty"` - Store *bool `json:"store,omitempty"` + ModelType agent_yaml.VoiceModelType `json:"modelType,omitempty"` + Model *agent_yaml.Model `json:"model,omitempty"` + Instructions *string `json:"instructions,omitempty"` + Voice *string `json:"voice,omitempty"` + StructuredInputs map[string]any `json:"structuredInputs,omitempty"` + Audio *agent_yaml.VoiceAudio `json:"audio,omitempty"` + OutputModalities []string `json:"outputModalities,omitempty"` + Store *bool `json:"store,omitempty"` + Tools []map[string]any `json:"tools,omitempty"` + Avatar map[string]any `json:"avatar,omitempty"` + Greeting map[string]any `json:"greeting,omitempty"` + Handoff map[string]any `json:"handoff,omitempty"` + ToolChoice any `json:"toolChoice,omitempty"` + ParallelToolCalls *bool `json:"parallelToolCalls,omitempty"` + MaxOutputTokens any `json:"maxOutputTokens,omitempty"` + Include []string `json:"include,omitempty"` } // voiceAgentDefinitionToInline projects a VoiceAgent into the inline definition // written to azure.yaml. Voice agents carry no container/image/code config. func voiceAgentDefinitionToInline(va agent_yaml.VoiceAgent) AgentDefinitionInline { return AgentDefinitionInline{ - AgentDefinition: va.AgentDefinition, - ModelType: va.ModelType, - Model: va.Model, - Instructions: va.Instructions, - Voice: va.Voice, - Store: va.Store, + AgentDefinition: va.AgentDefinition, + ModelType: va.ModelType, + Model: va.Model, + Instructions: va.Instructions, + Voice: va.Voice, + StructuredInputs: va.StructuredInputs, + Audio: va.Audio, + OutputModalities: va.OutputModalities, + Store: va.Store, + Tools: va.Tools, + Avatar: va.Avatar, + Greeting: va.Greeting, + Handoff: va.Handoff, + ToolChoice: va.ToolChoice, + ParallelToolCalls: va.ParallelToolCalls, + MaxOutputTokens: va.MaxOutputTokens, + Include: va.Include, } } // toVoiceAgent rebuilds an agent_yaml.VoiceAgent from the inline definition. func (d AgentDefinitionInline) toVoiceAgent() agent_yaml.VoiceAgent { return agent_yaml.VoiceAgent{ - AgentDefinition: d.AgentDefinition, - ModelType: d.ModelType, - Model: d.Model, - Instructions: d.Instructions, - Voice: d.Voice, - Store: d.Store, + AgentDefinition: d.AgentDefinition, + ModelType: d.ModelType, + Model: d.Model, + Instructions: d.Instructions, + Voice: d.Voice, + StructuredInputs: d.StructuredInputs, + Audio: d.Audio, + OutputModalities: d.OutputModalities, + Store: d.Store, + Tools: d.Tools, + Avatar: d.Avatar, + Greeting: d.Greeting, + Handoff: d.Handoff, + ToolChoice: d.ToolChoice, + ParallelToolCalls: d.ParallelToolCalls, + MaxOutputTokens: d.MaxOutputTokens, + Include: d.Include, } } diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go index a4558598fcb..eca8630b134 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go @@ -2223,6 +2223,13 @@ func (p *AgentServiceTargetProvider) deployVoiceAgent( fmt.Sprintf("set %s to legacy, unified, or unified-flat", voiceAgentAPIEnvKey), ) } + if hasAdvancedVoiceConfig(va) && apiMode != voiceAgentAPIModeUnifiedFlat { + return nil, exterrors.Validation( + exterrors.CodeInvalidParameter, + "advanced prompt-voice settings require the unified flat voice API mode", + fmt.Sprintf("set %s to unified-flat", voiceAgentAPIEnvKey), + ) + } var request *agent_api.CreateAgentRequest if apiMode == voiceAgentAPIModeUnifiedFlat { @@ -2299,6 +2306,20 @@ func (p *AgentServiceTargetProvider) deployVoiceAgent( return &azdext.ServiceDeployResult{Artifacts: artifacts}, nil } +func hasAdvancedVoiceConfig(va agent_yaml.VoiceAgent) bool { + return len(va.StructuredInputs) > 0 || + va.Audio != nil || + len(va.OutputModalities) > 0 || + len(va.Tools) > 0 || + len(va.Avatar) > 0 || + len(va.Greeting) > 0 || + len(va.Handoff) > 0 || + va.ToolChoice != nil || + va.ParallelToolCalls != nil || + va.MaxOutputTokens != nil || + len(va.Include) > 0 +} + func validateVoiceAgentDeployResponse(agentObject *agent_api.AgentObject, apiMode voiceAgentAPIMode) error { if agentObject == nil { return fmt.Errorf("malformed voice agent service response: missing agent object") diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go index 6c3fa573ed6..29c9c31ff06 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go @@ -748,6 +748,14 @@ func TestValidateVoiceAgentDeployResponse(t *testing.T) { }) } +func TestHasAdvancedVoiceConfig(t *testing.T) { + store := false + require.False(t, hasAdvancedVoiceConfig(agent_yaml.VoiceAgent{})) + require.False(t, hasAdvancedVoiceConfig(agent_yaml.VoiceAgent{Store: &store})) + require.True(t, hasAdvancedVoiceConfig(agent_yaml.VoiceAgent{Audio: &agent_yaml.VoiceAudio{}})) + require.True(t, hasAdvancedVoiceConfig(agent_yaml.VoiceAgent{Tools: []map[string]any{{"type": "system"}}})) +} + func createSymlinkOrSkip(t *testing.T, oldname, newname string) { t.Helper() diff --git a/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json b/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json index e486a47de8b..3c1cea4fc50 100644 --- a/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json +++ b/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json @@ -64,14 +64,62 @@ "type": "string", "description": "Voice agent (kind: prompt-voice) system prompt for the assistant." }, - "voice": { - "type": "string", - "description": "Voice agent (kind: prompt-voice) output voice name (e.g. 'en-US-Ava:DragonHDLatestNeural' for an Azure Neural voice, or 'alloy' for an OpenAI realtime voice)." - }, - "store": { - "type": "boolean", - "description": "Voice agent (kind: prompt-voice) server-side logging toggle (transcript + per-turn audio). Defaults to false when omitted." - }, + "voice": { + "type": "string", + "description": "Voice agent (kind: prompt-voice) output voice name (e.g. 'en-US-Ava:DragonHDLatestNeural' for an Azure Neural voice, or 'alloy' for an OpenAI realtime voice)." + }, + "structuredInputs": { + "type": "object", + "description": "Voice agent (kind: prompt-voice) structured prompt inputs. Use description, defaultValue, schema, and required; azd maps defaultValue to the service wire field default_value.", + "additionalProperties": true + }, + "audio": { + "$ref": "#/definitions/VoiceAudio" + }, + "outputModalities": { + "type": "array", + "description": "Voice agent (kind: prompt-voice) output modalities. Well-known values are audio, text, animation, and avatar.", + "items": { "type": "string", "minLength": 1 } + }, + "store": { + "type": "boolean", + "description": "Voice agent (kind: prompt-voice) server-side logging toggle (transcript + per-turn audio). Defaults to false when omitted." + }, + "tools": { + "type": "array", + "description": "Voice agent (kind: prompt-voice) tools. Direct tool types include function, mcp, system, and toolbox.", + "items": { "type": "object", "additionalProperties": true } + }, + "avatar": { + "type": "object", + "description": "Voice agent (kind: prompt-voice) avatar configuration.", + "additionalProperties": true + }, + "greeting": { + "type": "object", + "description": "Voice agent (kind: prompt-voice) greeting configuration, such as template or llm_generated.", + "additionalProperties": true + }, + "handoff": { + "type": "object", + "description": "Voice agent (kind: prompt-voice) handoff configuration.", + "additionalProperties": true + }, + "toolChoice": { + "description": "Voice agent (kind: prompt-voice) tool choice behavior, such as none, auto, required, or a tool choice object." + }, + "parallelToolCalls": { + "type": "boolean", + "description": "Voice agent (kind: prompt-voice) parallel tool call toggle." + }, + "maxOutputTokens": { + "description": "Voice agent (kind: prompt-voice) maximum output tokens. Use an integer or a service-supported string such as inf." + }, + "include": { + "type": "array", + "description": "Voice agent (kind: prompt-voice) extra service response fields to include.", + "items": { "type": "string" } + }, "name": { "type": "string", "description": "The agent name." @@ -152,7 +200,7 @@ "required": ["protocol"], "additionalProperties": false }, - "CodeConfiguration": { + "CodeConfiguration": { "type": "object", "description": "Code deploy configuration. When present, the agent is deployed from source (ZIP) instead of a container image.", "properties": { @@ -161,9 +209,96 @@ "dependencyResolution": { "type": "string", "description": "Dependency resolution mode (e.g., 'bundled', 'remote_build')." } }, "required": ["runtime", "entryPoint"], - "additionalProperties": false - }, - "SessionConfiguration": { + "additionalProperties": false + }, + "VoiceAudio": { + "type": "object", + "description": "Prompt voice input and output audio configuration. Requires AZURE_VOICE_AGENT_API=unified-flat for deployment.", + "properties": { + "input": { "$ref": "#/definitions/VoiceAudioInput" }, + "output": { "$ref": "#/definitions/VoiceAudioOutput" } + }, + "additionalProperties": false + }, + "VoiceAudioInput": { + "type": "object", + "properties": { + "format": { "$ref": "#/definitions/VoiceAudioFormat" }, + "noiseReduction": { "$ref": "#/definitions/VoiceNoiseReduction" }, + "echoCancellation": { "type": "object", "additionalProperties": true }, + "turnDetection": { "$ref": "#/definitions/VoiceTurnDetection" }, + "transcription": { "$ref": "#/definitions/VoiceTranscription" } + }, + "additionalProperties": false + }, + "VoiceAudioOutput": { + "type": "object", + "properties": { + "format": { "$ref": "#/definitions/VoiceAudioFormat" }, + "voice": { "$ref": "#/definitions/VoiceConfig" }, + "speed": { "type": "number", "minimum": 0.25, "maximum": 1.5 } + }, + "additionalProperties": false + }, + "VoiceAudioFormat": { + "type": "object", + "properties": { + "type": { "type": "string", "enum": ["audio/pcm", "audio/pcmu", "audio/pcma"] }, + "rate": { "type": "integer", "minimum": 1 } + }, + "required": ["type"], + "additionalProperties": false + }, + "VoiceNoiseReduction": { + "type": "object", + "properties": { + "type": { "type": "string", "description": "Well-known values include near_field, far_field, and azure_deep_noise_suppression." } + }, + "required": ["type"], + "additionalProperties": false + }, + "VoiceTurnDetection": { + "type": "object", + "properties": { + "type": { "type": "string", "description": "Well-known values include server_vad, semantic_vad, and azure_semantic_vad." }, + "threshold": { "type": "number", "minimum": 0, "maximum": 1 }, + "prefixPaddingMs": { "type": "integer", "minimum": 0 }, + "silenceDurationMs": { "type": "integer", "minimum": 0 }, + "createResponse": { "type": "boolean" }, + "eagerness": { "type": "string" }, + "speechDurationMs": { "type": "integer", "minimum": 0 }, + "removeFillerWords": { "type": "boolean" }, + "interruptResponse": { "type": "boolean" }, + "languages": { "type": "array", "items": { "type": "string" } }, + "autoTruncate": { "type": "boolean" } + }, + "required": ["type"], + "additionalProperties": false + }, + "VoiceTranscription": { + "type": "object", + "properties": { + "model": { "type": "string" }, + "language": { "type": "string" }, + "prompt": { "type": "string" } + }, + "additionalProperties": false + }, + "VoiceConfig": { + "type": "object", + "properties": { + "type": { "type": "string", "description": "Voice provider type, such as openai, azure_standard, or azure-standard." }, + "name": { "type": "string" }, + "style": { "type": "string" }, + "pitch": { "type": "string" }, + "rate": { "type": "string" }, + "locale": { "type": "string" }, + "volume": { "type": "string" } + }, + "required": ["type", "name"], + "additionalProperties": false + }, + "SessionConfiguration": { "type": "object", "description": "Optional hosted-agent session runtime settings. When omitted, the service applies its defaults (idle timeout 900 seconds).", "properties": { From ab7cd2551a4c518718fd89666893ad1136c65986 Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Thu, 20 Aug 2026 16:03:51 +0800 Subject: [PATCH 07/38] test(agents): avoid go fix pointer helper --- .../internal/pkg/agents/agent_yaml/map_voice_test.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go index 115538c572b..be279ddaec7 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go @@ -10,8 +10,6 @@ import ( "azureaiagent/internal/pkg/agents/agent_api" ) -func ptr[T any](v T) *T { return &v } - // --------------------------------------------------------------------------- // isOpenAIVoice / buildVoiceConfig // --------------------------------------------------------------------------- @@ -283,10 +281,13 @@ func TestCreateVoiceAgentAPIRequestFlat_AdvancedSettingsWireShape(t *testing.T) pitch := "+0Hz" rate := "+0%" volume := "+0%" + instructions := "You are {{persona}}, a concise voice assistant." + language := "en-US" + prompt := "Contoso terms" agent := VoiceAgent{ AgentDefinition: AgentDefinition{Kind: AgentKindPromptVoice, Name: "voice-advanced"}, Model: &Model{Id: "gpt-realtime"}, - Instructions: ptr("You are {{persona}}, a concise voice assistant."), + Instructions: &instructions, StructuredInputs: map[string]any{ "persona": map[string]any{"description": "Assistant persona", "defaultValue": "Ada"}, }, @@ -305,13 +306,13 @@ func TestCreateVoiceAgentAPIRequestFlat_AdvancedSettingsWireShape(t *testing.T) Languages: []string{"en-US"}, AutoTruncate: &autoTruncate, }, - Transcription: &VoiceTranscription{Model: "whisper-1", Language: ptr("en-US"), Prompt: ptr("Contoso terms")}, + Transcription: &VoiceTranscription{Model: "whisper-1", Language: &language, Prompt: &prompt}, }, Output: &VoiceAudioOutput{ Format: &VoiceAudioFormat{Type: "audio/pcm", Rate: &outRate}, Voice: &VoiceConfig{ Type: "azure_standard", Name: "en-US-AvaNeural", Style: &style, - Pitch: &pitch, Rate: &rate, Locale: ptr("en-US"), Volume: &volume, + Pitch: &pitch, Rate: &rate, Locale: &language, Volume: &volume, }, Speed: &speed, }, From ec43ae12bc273acf1c8a5a00e93a1ee79136e956 Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Thu, 20 Aug 2026 20:55:58 +0800 Subject: [PATCH 08/38] feat(agents): deploy hosted voice wrappers --- cli/azd/extensions/azure.ai.agents/README.md | 54 +++++++ .../internal/pkg/agents/agent_api/models.go | 69 +++++---- .../internal/pkg/agents/agent_yaml/map.go | 58 +++++-- .../pkg/agents/agent_yaml/map_voice_test.go | 57 +++++++ .../internal/pkg/agents/agent_yaml/parse.go | 33 +++- .../pkg/agents/agent_yaml/parse_voice_test.go | 44 ++++++ .../internal/pkg/agents/agent_yaml/yaml.go | 11 ++ .../internal/project/agent_definition.go | 41 +++-- .../internal/project/doc_examples_test.go | 1 + .../internal/project/foundry_dependencies.go | 66 ++++++++ .../project/hosted_voice_target_test.go | 146 ++++++++++++++++++ .../internal/project/service_target_agent.go | 116 +++++++++++++- .../project/service_target_agent_test.go | 27 ++++ .../schemas/azure.ai.agent.json | 63 ++++++-- 14 files changed, 703 insertions(+), 83 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.agents/internal/project/hosted_voice_target_test.go diff --git a/cli/azd/extensions/azure.ai.agents/README.md b/cli/azd/extensions/azure.ai.agents/README.md index e32553da8d9..02440aa86b8 100644 --- a/cli/azd/extensions/azure.ai.agents/README.md +++ b/cli/azd/extensions/azure.ai.agents/README.md @@ -164,6 +164,60 @@ Details: still require `legacy` or `unified` until the flat output shape is fully rolled out. +### Hosted voice wrapper (preview) + +A hosted voice wrapper keeps Voice Live responsible for VAD, speech-to-text, +and text-to-speech while routing conversation logic to a hosted agent in the +same Foundry project. Declare both services and reference the target by its +`azure.yaml` service name: + +```yaml +services: + voice-target: + host: azure.ai.agent + project: ./src/voice-target + language: csharp + kind: hosted + name: voice-target + protocols: + - protocol: invocations_ws + version: 1.0.0 + metadata: + voiceLiveCompatible: "true" + bridgeProtocolVersion: "1.0" + codeConfiguration: + runtime: dotnet_10 + entryPoint: VoiceHostedAgent.dll + dependencyResolution: bundled + + voice: + host: azure.ai.agent + kind: prompt-voice + name: voice + uses: + - voice-target + modelType: hosted_agent + targetAgent: + service: voice-target + version: deployed + store: false + audio: + output: + voice: + type: azure_standard + name: en-US-JennyNeural +``` + +The `uses` edge deploys the target before the wrapper. `version: deployed` +pins the wrapper to the target version produced by the current azd environment. +Hosted voice wrappers automatically use the unified flat Voice API unless +`AZURE_VOICE_AGENT_API` is explicitly set to an incompatible mode. + +The target must be active, declare `invocations_ws/1.0.0`, and include +`voiceLiveCompatible=true` and `bridgeProtocolVersion=1.0` metadata. Model, +instructions, tools, and other conversation controls belong to the target; +the wrapper owns audio, voice, store, avatar, and greeting configuration. + ## Prompt voice advanced configuration Advanced prompt voice settings are authored on the `azure.ai.agent` service in diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go index b278ab2ec64..0d4950747ce 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go @@ -307,8 +307,15 @@ type VoiceModelType string const ( VoiceModelTypeManaged VoiceModelType = "managed" VoiceModelTypeSelfDeployed VoiceModelType = "self_deployed" + VoiceModelTypeHostedAgent VoiceModelType = "hosted_agent" ) +// VoiceTargetAgentReference pins a voice wrapper to a hosted agent version. +type VoiceTargetAgentReference struct { + Name string `json:"name"` + Version string `json:"version,omitempty"` +} + // VoiceAudioFormat describes a PCM audio stream format (e.g. audio/pcm @ 24 kHz). type VoiceAudioFormat struct { Type string `json:"type"` @@ -404,21 +411,22 @@ type VoiceAudioConfigFlat struct { // is always AgentKindVoice ("voice"). type VoiceAgentDefinition struct { AgentDefinition - ModelType VoiceModelType `json:"model_type"` - Model string `json:"model"` - Instructions string `json:"instructions,omitempty"` - StructuredInputs map[string]any `json:"structured_inputs,omitempty"` - Audio *VoiceAudioConfig `json:"audio,omitempty"` - OutputModalities []string `json:"output_modalities,omitempty"` - Store *bool `json:"store,omitempty"` - Tools []map[string]any `json:"tools,omitempty"` - Avatar map[string]any `json:"avatar,omitempty"` - Greeting map[string]any `json:"greeting,omitempty"` - Handoff map[string]any `json:"handoff,omitempty"` - ToolChoice any `json:"tool_choice,omitempty"` - ParallelToolCalls *bool `json:"parallel_tool_calls,omitempty"` - MaxOutputTokens any `json:"max_output_tokens,omitempty"` - Include []string `json:"include,omitempty"` + ModelType VoiceModelType `json:"model_type"` + Model string `json:"model,omitempty"` + TargetAgent *VoiceTargetAgentReference `json:"target_agent,omitempty"` + Instructions string `json:"instructions,omitempty"` + StructuredInputs map[string]any `json:"structured_inputs,omitempty"` + Audio *VoiceAudioConfig `json:"audio,omitempty"` + OutputModalities []string `json:"output_modalities,omitempty"` + Store *bool `json:"store,omitempty"` + Tools []map[string]any `json:"tools,omitempty"` + Avatar map[string]any `json:"avatar,omitempty"` + Greeting map[string]any `json:"greeting,omitempty"` + Handoff map[string]any `json:"handoff,omitempty"` + ToolChoice any `json:"tool_choice,omitempty"` + ParallelToolCalls *bool `json:"parallel_tool_calls,omitempty"` + MaxOutputTokens any `json:"max_output_tokens,omitempty"` + Include []string `json:"include,omitempty"` } // VoiceAgentDefinitionFlat is the voice definition shape aligned with the @@ -426,21 +434,22 @@ type VoiceAgentDefinition struct { // provider details are sibling fields. type VoiceAgentDefinitionFlat struct { AgentDefinition - ModelType VoiceModelType `json:"model_type"` - Model string `json:"model"` - Instructions string `json:"instructions,omitempty"` - StructuredInputs map[string]any `json:"structured_inputs,omitempty"` - Audio *VoiceAudioConfigFlat `json:"audio,omitempty"` - OutputModalities []string `json:"output_modalities,omitempty"` - Store *bool `json:"store,omitempty"` - Tools []map[string]any `json:"tools,omitempty"` - Avatar map[string]any `json:"avatar,omitempty"` - Greeting map[string]any `json:"greeting,omitempty"` - Handoff map[string]any `json:"handoff,omitempty"` - ToolChoice any `json:"tool_choice,omitempty"` - ParallelToolCalls *bool `json:"parallel_tool_calls,omitempty"` - MaxOutputTokens any `json:"max_output_tokens,omitempty"` - Include []string `json:"include,omitempty"` + ModelType VoiceModelType `json:"model_type"` + Model string `json:"model,omitempty"` + TargetAgent *VoiceTargetAgentReference `json:"target_agent,omitempty"` + Instructions string `json:"instructions,omitempty"` + StructuredInputs map[string]any `json:"structured_inputs,omitempty"` + Audio *VoiceAudioConfigFlat `json:"audio,omitempty"` + OutputModalities []string `json:"output_modalities,omitempty"` + Store *bool `json:"store,omitempty"` + Tools []map[string]any `json:"tools,omitempty"` + Avatar map[string]any `json:"avatar,omitempty"` + Greeting map[string]any `json:"greeting,omitempty"` + Handoff map[string]any `json:"handoff,omitempty"` + ToolChoice any `json:"tool_choice,omitempty"` + ParallelToolCalls *bool `json:"parallel_tool_calls,omitempty"` + MaxOutputTokens any `json:"max_output_tokens,omitempty"` + Include []string `json:"include,omitempty"` } // CreateAgentVersionRequest represents a request to create an agent version diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go index f017c4875de..106cce4a483 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go @@ -698,36 +698,62 @@ func mapVoiceStructuredInputs(inputs map[string]any) map[string]any { // voice agent. It translates the authoring kind "prompt-voice" into the // data-plane service kind "voice" and defaults the audio pipeline. func CreateVoiceAgentAPIRequest(voiceAgent VoiceAgent) (*agent_api.CreateAgentRequest, error) { - return createVoiceAgentAPIRequest(voiceAgent, false) + return createVoiceAgentAPIRequest(voiceAgent, false, nil) } // CreateVoiceAgentAPIRequestFlat builds a CreateAgentRequest using the newer // TiP/unified API flat output voice shape. func CreateVoiceAgentAPIRequestFlat(voiceAgent VoiceAgent) (*agent_api.CreateAgentRequest, error) { - return createVoiceAgentAPIRequest(voiceAgent, true) + return createVoiceAgentAPIRequest(voiceAgent, true, nil) } -func createVoiceAgentAPIRequest(voiceAgent VoiceAgent, flatOutput bool) (*agent_api.CreateAgentRequest, error) { - modelID := "" - if voiceAgent.Model != nil { - modelID = strings.TrimSpace(voiceAgent.Model.Id) - } - if modelID == "" { - return nil, fmt.Errorf("model.id is required for a prompt-voice agent") - } +// CreateHostedVoiceAgentAPIRequestFlat builds a hosted-agent voice wrapper using +// the deployed target resolved by the project layer. +func CreateHostedVoiceAgentAPIRequestFlat( + voiceAgent VoiceAgent, + target agent_api.VoiceTargetAgentReference, +) (*agent_api.CreateAgentRequest, error) { + return createVoiceAgentAPIRequest(voiceAgent, true, &target) +} +func createVoiceAgentAPIRequest( + voiceAgent VoiceAgent, + flatOutput bool, + target *agent_api.VoiceTargetAgentReference, +) (*agent_api.CreateAgentRequest, error) { modelType := agent_api.VoiceModelTypeManaged if voiceAgent.ModelType != "" { modelType = agent_api.VoiceModelType(voiceAgent.ModelType) } - if modelType != agent_api.VoiceModelTypeManaged && modelType != agent_api.VoiceModelTypeSelfDeployed { + hostedAgent := modelType == agent_api.VoiceModelTypeHostedAgent + if hostedAgent { + if target == nil || strings.TrimSpace(target.Name) == "" || strings.TrimSpace(target.Version) == "" { + return nil, fmt.Errorf("resolved target agent name and version are required when model_type is 'hosted_agent'") + } + if voiceAgent.Model != nil || voiceAgent.Instructions != nil || len(voiceAgent.StructuredInputs) > 0 || + len(voiceAgent.Tools) > 0 || voiceAgent.ToolChoice != nil || voiceAgent.ParallelToolCalls != nil || + voiceAgent.MaxOutputTokens != nil || len(voiceAgent.Include) > 0 || len(voiceAgent.Handoff) > 0 { + return nil, fmt.Errorf("model, instructions, structured_inputs, tools, tool_choice, parallel_tool_calls, max_output_tokens, include, and handoff belong to the target hosted agent") + } + } else if modelType != agent_api.VoiceModelTypeManaged && modelType != agent_api.VoiceModelTypeSelfDeployed { return nil, fmt.Errorf( - "model_type '%s' is not supported; use '%s' or '%s'", - voiceAgent.ModelType, VoiceModelTypeManaged, VoiceModelTypeSelfDeployed) + "model_type '%s' is not supported; use '%s', '%s', or '%s'", + voiceAgent.ModelType, VoiceModelTypeManaged, VoiceModelTypeSelfDeployed, VoiceModelTypeHostedAgent) } - instructions := defaultVoiceInstructions - if voiceAgent.Instructions != nil && *voiceAgent.Instructions != "" { + modelID := "" + if voiceAgent.Model != nil { + modelID = strings.TrimSpace(voiceAgent.Model.Id) + } + if !hostedAgent && modelID == "" { + return nil, fmt.Errorf("model.id is required for a prompt-voice agent") + } + + instructions := "" + if !hostedAgent { + instructions = defaultVoiceInstructions + } + if !hostedAgent && voiceAgent.Instructions != nil && *voiceAgent.Instructions != "" { instructions = *voiceAgent.Instructions } @@ -781,6 +807,7 @@ func createVoiceAgentAPIRequest(voiceAgent VoiceAgent, flatOutput bool) (*agent_ }, ModelType: modelType, Model: modelID, + TargetAgent: target, Instructions: instructions, StructuredInputs: mapVoiceStructuredInputs(voiceAgent.StructuredInputs), Audio: &agent_api.VoiceAudioConfigFlat{ @@ -819,6 +846,7 @@ func createVoiceAgentAPIRequest(voiceAgent VoiceAgent, flatOutput bool) (*agent_ }, ModelType: modelType, Model: modelID, + TargetAgent: target, Instructions: instructions, StructuredInputs: mapVoiceStructuredInputs(voiceAgent.StructuredInputs), Audio: &agent_api.VoiceAudioConfig{ diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go index be279ddaec7..72128d67f0f 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go @@ -453,3 +453,60 @@ func TestCreateVoiceAgentAPIRequest_InvalidModelType(t *testing.T) { t.Error("expected error for unsupported model_type") } } + +func TestCreateHostedVoiceAgentAPIRequestFlat(t *testing.T) { + t.Parallel() + store := false + agent := VoiceAgent{ + AgentDefinition: AgentDefinition{Kind: AgentKindPromptVoice, Name: "voice-wrapper"}, + ModelType: VoiceModelTypeHostedAgent, + TargetAgent: &VoiceTargetAgent{Service: "voice-target", Version: "deployed"}, + Store: &store, + } + req, err := CreateHostedVoiceAgentAPIRequestFlat(agent, agent_api.VoiceTargetAgentReference{ + Name: "deployed-target", + Version: "7", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + def := req.Definition.(agent_api.VoiceAgentDefinitionFlat) + if def.ModelType != agent_api.VoiceModelTypeHostedAgent { + t.Fatalf("ModelType = %q, want hosted_agent", def.ModelType) + } + if def.TargetAgent == nil || def.TargetAgent.Name != "deployed-target" || def.TargetAgent.Version != "7" { + t.Fatalf("TargetAgent = %+v", def.TargetAgent) + } + if def.Model != "" || def.Instructions != "" || len(def.Tools) != 0 { + t.Fatalf("hosted wrapper contains target-owned fields: %+v", def) + } + data, err := json.Marshal(req) + if err != nil { + t.Fatal(err) + } + var wire map[string]any + if err := json.Unmarshal(data, &wire); err != nil { + t.Fatal(err) + } + definition := wire["definition"].(map[string]any) + if _, exists := definition["model"]; exists { + t.Fatalf("hosted wrapper wire payload contains model: %s", data) + } + if _, exists := definition["instructions"]; exists { + t.Fatalf("hosted wrapper wire payload contains instructions: %s", data) + } + if _, exists := definition["tools"]; exists { + t.Fatalf("hosted wrapper wire payload contains tools: %s", data) + } +} + +func TestCreateHostedVoiceAgentAPIRequestFlatRequiresResolvedTarget(t *testing.T) { + t.Parallel() + agent := VoiceAgent{ + AgentDefinition: AgentDefinition{Kind: AgentKindPromptVoice, Name: "voice-wrapper"}, + ModelType: VoiceModelTypeHostedAgent, + } + if _, err := CreateHostedVoiceAgentAPIRequestFlat(agent, agent_api.VoiceTargetAgentReference{}); err == nil { + t.Fatal("expected missing resolved target error") + } +} diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go index 250ca6776d1..54c4493c75b 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go @@ -430,15 +430,34 @@ func ValidateAgentDefinition(templateBytes []byte) error { case AgentKindPromptVoice: var agent VoiceAgent if err := yaml.Unmarshal(templateBytes, &agent); err == nil { - if agent.Model == nil || strings.TrimSpace(agent.Model.Id) == "" { - errors = append(errors, "template.model.id is required for a prompt-voice agent") + if agent.ModelType == VoiceModelTypeHostedAgent { + if agent.TargetAgent == nil || strings.TrimSpace(agent.TargetAgent.Service) == "" { + errors = append(errors, "template.target_agent.service is required when model_type is 'hosted_agent'") + } + if agent.TargetAgent != nil && agent.TargetAgent.Version != "" && agent.TargetAgent.Version != "deployed" { + errors = append(errors, "template.target_agent.version must be 'deployed' when specified") + } + if agent.Model != nil { + errors = append(errors, "template.model is not allowed when model_type is 'hosted_agent'") + } + if agent.Instructions != nil || len(agent.StructuredInputs) > 0 || len(agent.Tools) > 0 || + agent.ToolChoice != nil || agent.ParallelToolCalls != nil || agent.MaxOutputTokens != nil || + len(agent.Include) > 0 || len(agent.Handoff) > 0 { + errors = append(errors, "instructions, structured_inputs, tools, tool_choice, parallel_tool_calls, max_output_tokens, include, and handoff belong to the target hosted agent") + } + } else { + if agent.Model == nil || strings.TrimSpace(agent.Model.Id) == "" { + errors = append(errors, "template.model.id is required for a prompt-voice agent") + } + if agent.TargetAgent != nil { + errors = append(errors, "template.target_agent is only valid when model_type is 'hosted_agent'") + } } - if agent.ModelType != "" && - agent.ModelType != VoiceModelTypeManaged && - agent.ModelType != VoiceModelTypeSelfDeployed { + if agent.ModelType != "" && agent.ModelType != VoiceModelTypeManaged && + agent.ModelType != VoiceModelTypeSelfDeployed && agent.ModelType != VoiceModelTypeHostedAgent { errors = append(errors, fmt.Sprintf( - "template.model_type '%s' is not supported; use '%s' or '%s'", - agent.ModelType, VoiceModelTypeManaged, VoiceModelTypeSelfDeployed)) + "template.model_type '%s' is not supported; use '%s', '%s', or '%s'", + agent.ModelType, VoiceModelTypeManaged, VoiceModelTypeSelfDeployed, VoiceModelTypeHostedAgent)) } errors = append(errors, validateVoiceAgentAdvancedConfig(agent)...) } else { diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse_voice_test.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse_voice_test.go index 6aba8a368dd..c2bdc9be204 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse_voice_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse_voice_test.go @@ -118,3 +118,47 @@ model_type: unsupported t.Fatalf("expected invalid model_type error, got: %v", err) } } + +func TestValidateAgentDefinition_HostedVoiceAccepted(t *testing.T) { + yamlContent := []byte(` +kind: prompt-voice +name: voice-wrapper +model_type: hosted_agent +target_agent: + service: voice-target + version: deployed +`) + if err := ValidateAgentDefinition(yamlContent); err != nil { + t.Fatalf("expected hosted voice definition to be valid, got: %v", err) + } +} + +func TestValidateAgentDefinition_HostedVoiceRequiresTarget(t *testing.T) { + yamlContent := []byte(` +kind: prompt-voice +name: voice-wrapper +model_type: hosted_agent +`) + err := ValidateAgentDefinition(yamlContent) + if err == nil || !strings.Contains(err.Error(), "target_agent.service is required") { + t.Fatalf("expected target agent validation error, got: %v", err) + } +} + +func TestValidateAgentDefinition_HostedVoiceRejectsTargetOwnedFields(t *testing.T) { + yamlContent := []byte(` +kind: prompt-voice +name: voice-wrapper +model_type: hosted_agent +target_agent: + service: voice-target +model: + id: gpt-realtime +instructions: not allowed +`) + err := ValidateAgentDefinition(yamlContent) + if err == nil || !strings.Contains(err.Error(), "belong to the target hosted agent") || + !strings.Contains(err.Error(), "model is not allowed") { + t.Fatalf("expected target-owned field validation errors, got: %v", err) + } +} diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/yaml.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/yaml.go index 9d55803256e..37326f973bc 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/yaml.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/yaml.go @@ -30,8 +30,17 @@ type VoiceModelType string const ( VoiceModelTypeManaged VoiceModelType = "managed" VoiceModelTypeSelfDeployed VoiceModelType = "self_deployed" + VoiceModelTypeHostedAgent VoiceModelType = "hosted_agent" ) +// VoiceTargetAgent identifies the hosted agent service that supplies the +// conversation logic for a hosted voice wrapper. Service is an azure.yaml +// service name; azd resolves it to the deployed Foundry agent name and version. +type VoiceTargetAgent struct { + Service string `json:"service" yaml:"service"` + Version string `json:"version,omitempty" yaml:"version,omitempty"` +} + // IsValidAgentKind checks if the provided AgentKind is valid func IsValidAgentKind(kind AgentKind) bool { return slices.Contains(ValidAgentKinds(), kind) @@ -209,6 +218,8 @@ type VoiceAgent struct { // Model names the speech-to-speech model (e.g. "gpt-realtime"). Reuses the // shared Model struct; only Id is required for voice. Model *Model `json:"model,omitempty" yaml:"model,omitempty"` + // TargetAgent references the hosted agent service used when model_type is hosted_agent. + TargetAgent *VoiceTargetAgent `json:"targetAgent,omitempty" yaml:"target_agent,omitempty"` // Instructions is the system prompt for the voice assistant. Instructions *string `json:"instructions,omitempty" yaml:"instructions,omitempty"` // Voice is the output voice name (e.g. "en-US-Ava:DragonHDLatestNeural" for diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/agent_definition.go b/cli/azd/extensions/azure.ai.agents/internal/project/agent_definition.go index 1625d9a7637..10242387b71 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/agent_definition.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/agent_definition.go @@ -138,22 +138,23 @@ type AgentDefinitionInline struct { // Voice-agent fields (kind: prompt-voice). All omitempty so container/ // workflow entries are byte-for-byte unchanged. - ModelType agent_yaml.VoiceModelType `json:"modelType,omitempty"` - Model *agent_yaml.Model `json:"model,omitempty"` - Instructions *string `json:"instructions,omitempty"` - Voice *string `json:"voice,omitempty"` - StructuredInputs map[string]any `json:"structuredInputs,omitempty"` - Audio *agent_yaml.VoiceAudio `json:"audio,omitempty"` - OutputModalities []string `json:"outputModalities,omitempty"` - Store *bool `json:"store,omitempty"` - Tools []map[string]any `json:"tools,omitempty"` - Avatar map[string]any `json:"avatar,omitempty"` - Greeting map[string]any `json:"greeting,omitempty"` - Handoff map[string]any `json:"handoff,omitempty"` - ToolChoice any `json:"toolChoice,omitempty"` - ParallelToolCalls *bool `json:"parallelToolCalls,omitempty"` - MaxOutputTokens any `json:"maxOutputTokens,omitempty"` - Include []string `json:"include,omitempty"` + ModelType agent_yaml.VoiceModelType `json:"modelType,omitempty"` + Model *agent_yaml.Model `json:"model,omitempty"` + TargetAgent *agent_yaml.VoiceTargetAgent `json:"targetAgent,omitempty"` + Instructions *string `json:"instructions,omitempty"` + Voice *string `json:"voice,omitempty"` + StructuredInputs map[string]any `json:"structuredInputs,omitempty"` + Audio *agent_yaml.VoiceAudio `json:"audio,omitempty"` + OutputModalities []string `json:"outputModalities,omitempty"` + Store *bool `json:"store,omitempty"` + Tools []map[string]any `json:"tools,omitempty"` + Avatar map[string]any `json:"avatar,omitempty"` + Greeting map[string]any `json:"greeting,omitempty"` + Handoff map[string]any `json:"handoff,omitempty"` + ToolChoice any `json:"toolChoice,omitempty"` + ParallelToolCalls *bool `json:"parallelToolCalls,omitempty"` + MaxOutputTokens any `json:"maxOutputTokens,omitempty"` + Include []string `json:"include,omitempty"` } // voiceAgentDefinitionToInline projects a VoiceAgent into the inline definition @@ -163,6 +164,7 @@ func voiceAgentDefinitionToInline(va agent_yaml.VoiceAgent) AgentDefinitionInlin AgentDefinition: va.AgentDefinition, ModelType: va.ModelType, Model: va.Model, + TargetAgent: va.TargetAgent, Instructions: va.Instructions, Voice: va.Voice, StructuredInputs: va.StructuredInputs, @@ -186,6 +188,7 @@ func (d AgentDefinitionInline) toVoiceAgent() agent_yaml.VoiceAgent { AgentDefinition: d.AgentDefinition, ModelType: d.ModelType, Model: d.Model, + TargetAgent: d.TargetAgent, Instructions: d.Instructions, Voice: d.Voice, StructuredInputs: d.StructuredInputs, @@ -782,7 +785,11 @@ func agentDefinitionFromStruct( } if inline.Kind != agent_yaml.AgentKindHosted { - if err := validateAgentServiceDefinition(s.AsMap()); err != nil { + definition := any(s.AsMap()) + if inline.Kind == agent_yaml.AgentKindPromptVoice { + definition = inline.toVoiceAgent() + } + if err := validateAgentServiceDefinition(definition); err != nil { return agent_yaml.ContainerAgent{}, false, err } return agent_yaml.ContainerAgent{}, false, nil diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/doc_examples_test.go b/cli/azd/extensions/azure.ai.agents/internal/project/doc_examples_test.go index d0260148930..e3240bb53c5 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/doc_examples_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/doc_examples_test.go @@ -548,6 +548,7 @@ func serviceConfigFromDoc(t *testing.T, e docExample, name string, svc map[strin RelativePath: core.RelativePath, Image: core.Image, AdditionalProperties: props, + Uses: core.Uses, } // The deprecated shape nests the agent definition under `config`. diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/foundry_dependencies.go b/cli/azd/extensions/azure.ai.agents/internal/project/foundry_dependencies.go index 65faf5b730c..6bb27558887 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/foundry_dependencies.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/foundry_dependencies.go @@ -12,6 +12,7 @@ import ( "strings" "azureaiagent/internal/exterrors" + "azureaiagent/internal/pkg/agents/agent_yaml" "azureaiagent/internal/pkg/envkey" "github.com/azure/azure-dev/cli/azd/pkg/azdext" @@ -19,6 +20,13 @@ import ( type dependencyEnabled func(context.Context, string) (bool, error) +type hostedVoiceTarget struct { + ServiceName string + AgentName string + AgentVersion string + ProjectEndpoint string +} + const ( foundryProjectHost = "azure.ai.project" foundryConnectionHost = "azure.ai.connection" @@ -351,6 +359,64 @@ func validateFoundryAgentDependency(service *azdext.ServiceConfig, env map[strin return "" } +func resolveHostedVoiceTarget( + wrapper *azdext.ServiceConfig, + voiceAgentTarget *agent_yaml.VoiceTargetAgent, + services map[string]*azdext.ServiceConfig, + env map[string]string, + projectRoot string, +) (*hostedVoiceTarget, error) { + if voiceAgentTarget == nil || strings.TrimSpace(voiceAgentTarget.Service) == "" { + return nil, fmt.Errorf("targetAgent.service is required when modelType is hosted_agent") + } + targetServiceName := strings.TrimSpace(voiceAgentTarget.Service) + if !slices.Contains(wrapper.GetUses(), targetServiceName) { + return nil, fmt.Errorf( + "hosted voice target service %q must be declared in the %q service uses list", + targetServiceName, wrapper.GetName()) + } + targetService, ok := services[targetServiceName] + if !ok { + return nil, fmt.Errorf("hosted voice target service %q was not found in azure.yaml", targetServiceName) + } + if targetService.GetHost() != foundryAgentHost { + return nil, fmt.Errorf( + "hosted voice target service %q must use host %q, got %q", + targetServiceName, foundryAgentHost, targetService.GetHost()) + } + _, isHosted, _, err := LoadAgentDefinition(targetService, projectRoot) + if err != nil { + return nil, fmt.Errorf("loading hosted voice target service %q: %w", targetServiceName, err) + } + if !isHosted { + return nil, fmt.Errorf("hosted voice target service %q must have kind hosted", targetServiceName) + } + + key := normalizeAgentServiceKey(targetServiceName) + name := strings.TrimSpace(env[fmt.Sprintf("AGENT_%s_NAME", key)]) + version := strings.TrimSpace(env[fmt.Sprintf("AGENT_%s_VERSION", key)]) + projectEndpoint := strings.TrimSpace(env[envkey.AgentProjectEndpoint(targetServiceName)]) + baseEndpoint := strings.TrimSpace(env[fmt.Sprintf("AGENT_%s_ENDPOINT", key)]) + if projectEndpoint == "" && endpointBelongsToProject(baseEndpoint, env["FOUNDRY_PROJECT_ENDPOINT"]) { + projectEndpoint = strings.TrimRight(strings.TrimSpace(env["FOUNDRY_PROJECT_ENDPOINT"]), "/") + } + if name == "" || version == "" || projectEndpoint == "" { + return nil, fmt.Errorf( + "hosted voice target service %q is not deployed; run 'azd deploy %s' or 'azd deploy --all'", + targetServiceName, strconv.Quote(targetServiceName)) + } + if !sameProjectEndpoint(projectEndpoint, env["FOUNDRY_PROJECT_ENDPOINT"]) { + return nil, fmt.Errorf("hosted voice target service %q is deployed to a different Foundry project", targetServiceName) + } + + return &hostedVoiceTarget{ + ServiceName: targetServiceName, + AgentName: name, + AgentVersion: version, + ProjectEndpoint: projectEndpoint, + }, nil +} + func endpointBelongsToProject(resourceEndpoint, projectEndpoint string) bool { resourceEndpoint = strings.TrimRight(strings.TrimSpace(resourceEndpoint), "/") projectEndpoint = strings.TrimRight(strings.TrimSpace(projectEndpoint), "/") diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/hosted_voice_target_test.go b/cli/azd/extensions/azure.ai.agents/internal/project/hosted_voice_target_test.go new file mode 100644 index 00000000000..dcd445c29f8 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/internal/project/hosted_voice_target_test.go @@ -0,0 +1,146 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "testing" + + "azureaiagent/internal/pkg/agents/agent_api" + "azureaiagent/internal/pkg/agents/agent_yaml" + "azureaiagent/internal/pkg/envkey" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/require" +) + +func TestResolveHostedVoiceTarget(t *testing.T) { + targetProps, err := AgentDefinitionToServiceProperties(agent_yaml.ContainerAgent{ + AgentDefinition: agent_yaml.AgentDefinition{Kind: agent_yaml.AgentKindHosted, Name: "remote-target"}, + }, nil) + require.NoError(t, err) + target := &azdext.ServiceConfig{ + Name: "voice-target", + Host: foundryAgentHost, + AdditionalProperties: targetProps, + } + wrapper := &azdext.ServiceConfig{ + Name: "voice-wrapper", + Host: foundryAgentHost, + Uses: []string{"voice-target"}, + } + projectEndpoint := "https://account.services.ai.azure.com/api/projects/project" + env := map[string]string{ + "FOUNDRY_PROJECT_ENDPOINT": projectEndpoint, + "AGENT_VOICE_TARGET_NAME": "remote-target", + "AGENT_VOICE_TARGET_VERSION": "4", + envkey.AgentProjectEndpoint("voice-target"): projectEndpoint, + } + + resolved, err := resolveHostedVoiceTarget( + wrapper, + &agent_yaml.VoiceTargetAgent{Service: "voice-target", Version: "deployed"}, + map[string]*azdext.ServiceConfig{"voice-target": target}, + env, + t.TempDir(), + ) + require.NoError(t, err) + require.Equal(t, "remote-target", resolved.AgentName) + require.Equal(t, "4", resolved.AgentVersion) +} + +func TestResolveHostedVoiceTargetRequiresUses(t *testing.T) { + wrapper := &azdext.ServiceConfig{Name: "voice-wrapper", Host: foundryAgentHost} + _, err := resolveHostedVoiceTarget( + wrapper, + &agent_yaml.VoiceTargetAgent{Service: "voice-target"}, + map[string]*azdext.ServiceConfig{}, + map[string]string{}, + t.TempDir(), + ) + require.ErrorContains(t, err, "uses list") +} + +func TestResolveHostedVoiceTargetSupportsLegacyEndpointMarker(t *testing.T) { + targetProps, err := AgentDefinitionToServiceProperties(agent_yaml.ContainerAgent{ + AgentDefinition: agent_yaml.AgentDefinition{Kind: agent_yaml.AgentKindHosted, Name: "remote-target"}, + }, nil) + require.NoError(t, err) + target := &azdext.ServiceConfig{Name: "voice-target", Host: foundryAgentHost, AdditionalProperties: targetProps} + wrapper := &azdext.ServiceConfig{Name: "voice-wrapper", Host: foundryAgentHost, Uses: []string{"voice-target"}} + projectEndpoint := "https://account.services.ai.azure.com/api/projects/project" + resolved, err := resolveHostedVoiceTarget( + wrapper, + &agent_yaml.VoiceTargetAgent{Service: "voice-target"}, + map[string]*azdext.ServiceConfig{"voice-target": target}, + map[string]string{ + "FOUNDRY_PROJECT_ENDPOINT": projectEndpoint, + "AGENT_VOICE_TARGET_NAME": "remote-target", + "AGENT_VOICE_TARGET_VERSION": "4", + "AGENT_VOICE_TARGET_ENDPOINT": projectEndpoint + "/agents/remote-target/versions/4", + }, + t.TempDir(), + ) + require.NoError(t, err) + require.Equal(t, projectEndpoint, resolved.ProjectEndpoint) +} + +func TestResolveHostedVoiceTargetRejectsDifferentProject(t *testing.T) { + targetProps, err := AgentDefinitionToServiceProperties(agent_yaml.ContainerAgent{ + AgentDefinition: agent_yaml.AgentDefinition{Kind: agent_yaml.AgentKindHosted, Name: "remote-target"}, + }, nil) + require.NoError(t, err) + target := &azdext.ServiceConfig{Name: "voice-target", Host: foundryAgentHost, AdditionalProperties: targetProps} + wrapper := &azdext.ServiceConfig{Name: "voice-wrapper", Host: foundryAgentHost, Uses: []string{"voice-target"}} + _, err = resolveHostedVoiceTarget( + wrapper, + &agent_yaml.VoiceTargetAgent{Service: "voice-target"}, + map[string]*azdext.ServiceConfig{"voice-target": target}, + map[string]string{ + "FOUNDRY_PROJECT_ENDPOINT": "https://account.services.ai.azure.com/api/projects/current", + "AGENT_VOICE_TARGET_NAME": "remote-target", + "AGENT_VOICE_TARGET_VERSION": "4", + envkey.AgentProjectEndpoint("voice-target"): "https://account.services.ai.azure.com/api/projects/other", + }, + t.TempDir(), + ) + require.ErrorContains(t, err, "different Foundry project") +} + +func TestValidateHostedVoiceTarget(t *testing.T) { + err := validateHostedVoiceTargetVersion(&agent_api.AgentVersionObject{ + Name: "remote-target", + Version: "4", + Status: "active", + Metadata: map[string]string{ + "voiceLiveCompatible": "true", + "bridgeProtocolVersion": "1.0", + }, + Definition: map[string]any{ + "kind": "hosted", + "protocol_versions": []any{map[string]any{ + "protocol": "invocations_ws", + "version": "1.0.0", + }}, + }, + }) + require.NoError(t, err) +} + +func TestValidateHostedVoiceTargetRejectsIncompatibleProtocol(t *testing.T) { + err := validateHostedVoiceTargetVersion(&agent_api.AgentVersionObject{ + Status: "active", + Metadata: map[string]string{ + "voiceLiveCompatible": "true", + "bridgeProtocolVersion": "1.0", + }, + Definition: map[string]any{ + "kind": "hosted", + "protocol_versions": []any{map[string]any{ + "protocol": "responses", + "version": "2.0.0", + }}, + }, + }) + require.ErrorContains(t, err, "invocations_ws/1.0.0") +} diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go index eca8630b134..ce8bc43ac9e 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go @@ -10,6 +10,7 @@ import ( "crypto/sha256" "encoding/base64" "encoding/hex" + "encoding/json" "errors" "fmt" "io" @@ -2223,6 +2224,17 @@ func (p *AgentServiceTargetProvider) deployVoiceAgent( fmt.Sprintf("set %s to legacy, unified, or unified-flat", voiceAgentAPIEnvKey), ) } + if va.ModelType == agent_yaml.VoiceModelTypeHostedAgent && + strings.TrimSpace(azdEnv[voiceAgentAPIEnvKey]) == "" && strings.TrimSpace(os.Getenv(voiceAgentAPIEnvKey)) == "" { + apiMode = voiceAgentAPIModeUnifiedFlat + } + if va.ModelType == agent_yaml.VoiceModelTypeHostedAgent && apiMode != voiceAgentAPIModeUnifiedFlat { + return nil, exterrors.Validation( + exterrors.CodeInvalidParameter, + "hosted-agent voice wrappers require the unified flat voice API mode", + fmt.Sprintf("set %s to unified-flat", voiceAgentAPIEnvKey), + ) + } if hasAdvancedVoiceConfig(va) && apiMode != voiceAgentAPIModeUnifiedFlat { return nil, exterrors.Validation( exterrors.CodeInvalidParameter, @@ -2232,7 +2244,39 @@ func (p *AgentServiceTargetProvider) deployVoiceAgent( } var request *agent_api.CreateAgentRequest - if apiMode == voiceAgentAPIModeUnifiedFlat { + var hostedTarget *hostedVoiceTarget + if va.ModelType == agent_yaml.VoiceModelTypeHostedAgent { + if va.TargetAgent != nil && p.dependencyEnabled != nil { + enabled, enabledErr := p.dependencyEnabled(ctx, strings.TrimSpace(va.TargetAgent.Service)) + if enabledErr != nil { + return nil, enabledErr + } + if !enabled { + return nil, exterrors.Dependency( + exterrors.CodeFoundryDependencyNotReady, + fmt.Sprintf("hosted voice target service %q is disabled", va.TargetAgent.Service), + "enable the target hosted agent service or remove the voice wrapper", + ) + } + } + hostedTarget, err = resolveHostedVoiceTarget( + serviceConfig, va.TargetAgent, p.projectServices, azdEnv, p.projectPath, + ) + if err != nil { + return nil, exterrors.Dependency( + exterrors.CodeFoundryDependencyNotReady, + fmt.Sprintf("cannot resolve hosted voice target: %s", err), + "deploy the target hosted agent in the same project, then retry", + ) + } + request, err = agent_yaml.CreateHostedVoiceAgentAPIRequestFlat( + va, + agent_api.VoiceTargetAgentReference{ + Name: hostedTarget.AgentName, + Version: hostedTarget.AgentVersion, + }, + ) + } else if apiMode == voiceAgentAPIModeUnifiedFlat { request, err = agent_yaml.CreateVoiceAgentAPIRequestFlat(va) } else { request, err = agent_yaml.CreateVoiceAgentAPIRequest(va) @@ -2256,6 +2300,16 @@ func (p *AgentServiceTargetProvider) deployVoiceAgent( } agentClient := agent_api.NewAgentClient(projectEndpoint, p.credential) + if hostedTarget != nil { + progress("Validating hosted voice target") + if err := validateHostedVoiceTarget(ctx, agentClient, hostedTarget); err != nil { + return nil, exterrors.Dependency( + exterrors.CodeFoundryDependencyNotReady, + fmt.Sprintf("hosted voice target is not compatible: %s", err), + "deploy an active Voice Bridge 1.0 hosted agent with invocations_ws/1.0.0, then retry", + ) + } + } serviceKey := p.getServiceKey(serviceConfig.Name) agentObject, deployOp, err := p.deployVoiceAgentWithMode( @@ -2278,9 +2332,18 @@ func (p *AgentServiceTargetProvider) deployVoiceAgent( if apiMode != voiceAgentAPIModeLegacy { versionValue = agentObject.Versions.Latest.Version } + targetName := "" + targetVersion := "" + if hostedTarget != nil { + targetName = hostedTarget.AgentName + targetVersion = hostedTarget.AgentVersion + } for _, envVar := range []struct{ key, value string }{ {fmt.Sprintf("AGENT_%s_NAME", serviceKey), agentObject.Name}, {versionKey, versionValue}, + {fmt.Sprintf("AGENT_%s_PROJECT_ENDPOINT", serviceKey), strings.TrimRight(projectEndpoint, "/")}, + {fmt.Sprintf("AGENT_%s_TARGET_NAME", serviceKey), targetName}, + {fmt.Sprintf("AGENT_%s_TARGET_VERSION", serviceKey), targetVersion}, {fmt.Sprintf("AGENT_%s_ENDPOINT", serviceKey), baseEndpoint}, } { if _, setErr := p.azdClient.Environment().SetValue(ctx, &azdext.SetEnvRequest{ @@ -2306,6 +2369,57 @@ func (p *AgentServiceTargetProvider) deployVoiceAgent( return &azdext.ServiceDeployResult{Artifacts: artifacts}, nil } +func validateHostedVoiceTarget( + ctx context.Context, + agentClient *agent_api.AgentClient, + target *hostedVoiceTarget, +) error { + version, err := agentClient.GetAgentVersion( + ctx, target.AgentName, target.AgentVersion, agent_api.AgentEndpointAPIVersion, + ) + if err != nil { + return fmt.Errorf("getting target %s:%s: %w", target.AgentName, target.AgentVersion, err) + } + return validateHostedVoiceTargetVersion(version) +} + +func validateHostedVoiceTargetVersion(version *agent_api.AgentVersionObject) error { + if version == nil { + return fmt.Errorf("target version response is empty") + } + if version.Status != "active" { + return fmt.Errorf("target %s:%s has status %q, expected active", version.Name, version.Version, version.Status) + } + definitionJSON, err := json.Marshal(version.Definition) + if err != nil { + return fmt.Errorf("reading target definition: %w", err) + } + var definition agent_api.HostedAgentDefinition + if err := json.Unmarshal(definitionJSON, &definition); err != nil { + return fmt.Errorf("reading target definition: %w", err) + } + if definition.Kind != agent_api.AgentKindHosted { + return fmt.Errorf("target kind is %q, expected hosted", definition.Kind) + } + compatibleProtocol := false + for _, protocol := range definition.ProtocolVersions { + if protocol.Protocol == agent_api.AgentProtocolInvocationsWS && protocol.Version == "1.0.0" { + compatibleProtocol = true + break + } + } + if !compatibleProtocol { + return fmt.Errorf("target does not declare invocations_ws/1.0.0") + } + if !strings.EqualFold(strings.TrimSpace(version.Metadata["voiceLiveCompatible"]), "true") { + return fmt.Errorf("target metadata voiceLiveCompatible must be true") + } + if strings.TrimSpace(version.Metadata["bridgeProtocolVersion"]) != "1.0" { + return fmt.Errorf("target metadata bridgeProtocolVersion must be 1.0") + } + return nil +} + func hasAdvancedVoiceConfig(va agent_yaml.VoiceAgent) bool { return len(va.StructuredInputs) > 0 || va.Audio != nil || diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go index 29c9c31ff06..430a23bdf52 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go @@ -68,6 +68,33 @@ func TestVoiceAgentInlineServicePropertiesRoundTrip_BYOM(t *testing.T) { require.Equal(t, store, *got.Store) } +func TestVoiceAgentInlineServicePropertiesRoundTrip_HostedAgent(t *testing.T) { + props, err := VoiceAgentDefinitionToServiceProperties(agent_yaml.VoiceAgent{ + AgentDefinition: agent_yaml.AgentDefinition{ + Kind: agent_yaml.AgentKindPromptVoice, + Name: "voice-wrapper", + }, + ModelType: agent_yaml.VoiceModelTypeHostedAgent, + TargetAgent: &agent_yaml.VoiceTargetAgent{ + Service: "voice-target", + Version: "deployed", + }, + }, nil) + require.NoError(t, err) + + svc := &azdext.ServiceConfig{ + Name: "voice-wrapper", + Host: "azure.ai.agent", + AdditionalProperties: props, + } + got, found, err := VoiceAgentFromResolvedService(svc, t.TempDir()) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, agent_yaml.VoiceModelTypeHostedAgent, got.ModelType) + require.Equal(t, "voice-target", got.TargetAgent.Service) + require.Equal(t, "deployed", got.TargetAgent.Version) +} + func TestApplyAgentMetadata(t *testing.T) { tests := []struct { name string diff --git a/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json b/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json index 3c1cea4fc50..84ad94a6313 100644 --- a/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json +++ b/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json @@ -48,8 +48,11 @@ }, "modelType": { "type": "string", - "description": "Voice agent (kind: prompt-voice) model-inference mode. 'managed' uses a Voice Live-hosted model; 'self_deployed' (BYOM) references an existing Foundry model deployment.", - "enum": ["managed", "self_deployed"] + "description": "Voice agent model-inference mode. 'managed' uses a Voice Live-hosted model; 'self_deployed' references a Foundry model deployment; 'hosted_agent' routes turns to a deployed hosted agent service.", + "enum": ["managed", "self_deployed", "hosted_agent"] + }, + "targetAgent": { + "$ref": "#/definitions/VoiceTargetAgent" }, "model": { "type": "object", @@ -177,19 +180,53 @@ "additionalProperties": true, "allOf": [ { - "$comment": "A prompt-voice agent must declare a speech-to-speech model; the deploy path rejects a voice service whose model.id is missing. Keep editor/schema validation aligned with that runtime requirement.", - "if": { - "properties": { - "kind": { "const": "prompt-voice" } - }, - "required": ["kind"] - }, - "then": { - "required": ["model"] - } + "$comment": "A hosted-agent voice wrapper references a deployed hosted service; other voice modes declare a model.", + "if": { + "properties": { + "kind": { "const": "prompt-voice" }, + "modelType": { "const": "hosted_agent" } + }, + "required": ["kind", "modelType"] + }, + "then": { + "required": ["targetAgent"], + "not": { + "anyOf": [ + { "required": ["model"] }, + { "required": ["instructions"] }, + { "required": ["structuredInputs"] }, + { "required": ["tools"] }, + { "required": ["toolChoice"] }, + { "required": ["parallelToolCalls"] }, + { "required": ["maxOutputTokens"] }, + { "required": ["include"] }, + { "required": ["handoff"] } + ] + } + }, + "else": { + "if": { + "properties": { "kind": { "const": "prompt-voice" } }, + "required": ["kind"] + }, + "then": { + "required": ["model"], + "not": { "required": ["targetAgent"] } + } + } } ], - "definitions": { + "definitions": { + "VoiceTargetAgent": { + "type": "object", + "description": "Hosted agent service used as the conversation target for a voice wrapper.", + "properties": { + "service": { "type": "string", "minLength": 1, "pattern": "\\S", "description": "azure.yaml service name of the target hosted agent." }, + "version": { "type": "string", "enum": ["deployed"], "default": "deployed", "description": "Pin the wrapper to the target version deployed by this azd environment." } + }, + "required": ["service"], + "additionalProperties": false + }, "ProtocolVersionRecord": { "type": "object", "description": "A protocol the agent implements, with its version.", From 99cae8c7bdbeac5a7d4dacfa88f8992cd7ef3f1b Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Thu, 20 Aug 2026 22:34:21 +0800 Subject: [PATCH 09/38] docs(agents): add hosted voice test guide --- cli/azd/extensions/azure.ai.agents/README.md | 3 + .../docs/hosted-voice-test-guide.md | 324 ++++++++++++++++++ 2 files changed, 327 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.agents/docs/hosted-voice-test-guide.md diff --git a/cli/azd/extensions/azure.ai.agents/README.md b/cli/azd/extensions/azure.ai.agents/README.md index 02440aa86b8..43977202514 100644 --- a/cli/azd/extensions/azure.ai.agents/README.md +++ b/cli/azd/extensions/azure.ai.agents/README.md @@ -218,6 +218,9 @@ The target must be active, declare `invocations_ws/1.0.0`, and include instructions, tools, and other conversation controls belong to the target; the wrapper owns audio, voice, store, avatar, and greeting configuration. +For end-to-end validation, lifecycle checks, local dashboard steps, and the +experience alignment matrix, see [Hosted Voice Agent Test Guide](docs/hosted-voice-test-guide.md). + ## Prompt voice advanced configuration Advanced prompt voice settings are authored on the `azure.ai.agent` service in diff --git a/cli/azd/extensions/azure.ai.agents/docs/hosted-voice-test-guide.md b/cli/azd/extensions/azure.ai.agents/docs/hosted-voice-test-guide.md new file mode 100644 index 00000000000..495e3ae3569 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/docs/hosted-voice-test-guide.md @@ -0,0 +1,324 @@ +# Hosted Voice Agent Test Guide + +This guide validates the preview `azd` Hosted Voice Agent experience against an +existing Foundry project. It covers the same lifecycle surfaces used by hosted +code agents and `invocations_ws` agents, plus the Voice wrapper and local Voice +dashboard. + +## Architecture under test + +```text +Voice client / local dashboard + | + | Voice realtime protocol + v +Voice wrapper (kind=voice, model_type=hosted_agent) + | + | Voice Bridge Protocol 1.0 over invocations_ws/1.0.0 + v +Hosted target (kind=hosted, user code) +``` + +Users write and deploy the hosted target code. The wrapper owns VAD, STT, TTS, +output voice, and Voice session settings. The wrapper and target must be in the +same Foundry project. + +## Prerequisites + +- A Foundry project in a region where Hosted Voice is enabled. West US 2 is the + recommended preview validation region. +- A model deployment that the hosted target can invoke. +- `az login` and `azd auth login` completed for the test subscription. +- A local build of the PR extension installed: + + ```powershell + cd cli/azd/extensions/azure.ai.agents + azd x build + ``` + +- A Hosted Voice target implementing Voice Bridge Protocol 1.0. The current + .NET sample is under `samples/voice-hosted-agent/voice-hosted-agent-dotnet` in + the `voice-first-agent-dev` repository. + +## Manifest + +Use one project service, one hosted target, and one Voice wrapper. The wrapper +references the target by its `azure.yaml` service name, not by a remote agent +name copied into the file. + +```yaml +services: + ai-project: + host: azure.ai.project + + voice-target: + host: azure.ai.agent + project: ./src/voice-target + language: csharp + kind: hosted + name: voice-target + uses: + - ai-project + metadata: + voiceLiveCompatible: "true" + bridgeProtocolVersion: "1.0" + protocols: + - protocol: invocations_ws + version: 1.0.0 + codeConfiguration: + runtime: dotnet_10 + entryPoint: VoiceHostedAgent.dll + dependencyResolution: bundled + container: + resources: + cpu: "1" + memory: 2Gi + + voice: + host: azure.ai.agent + kind: prompt-voice + name: voice + uses: + - ai-project + - voice-target + modelType: hosted_agent + targetAgent: + service: voice-target + version: deployed + store: false + audio: + output: + voice: + type: azure_standard + name: en-US-JennyNeural +``` + +`version: deployed` pins the wrapper to the target version recorded by the +current azd environment. Floating latest is intentionally not supported. + +## Automated local checks + +Run from `cli/azd/extensions/azure.ai.agents`: + +```powershell +go test ./... +go vet ./... +azd x build +``` + +Expected: all commands succeed. + +## Package and publish + +Run from the test azd project: + +```powershell +azd package --all +azd publish --all +``` + +Expected: + +- the hosted target builds and produces a code ZIP or container artifact; +- the project and Voice wrapper report no package artifact; +- publish succeeds without attempting to publish wrapper code. + +This matches the existing split between a code agent service and a declarative +service resource. + +## Deploy + +```powershell +azd deploy --all --no-prompt +``` + +Expected ordering: + +1. project dependency is ready; +2. target is packaged and deployed; +3. target version becomes active; +4. wrapper validates the target; +5. wrapper is created or updated through the unified Voice API. + +Expected target validation: + +- same Foundry project; +- `kind=hosted`; +- status `active`; +- `invocations_ws/1.0.0`; +- metadata `voiceLiveCompatible=true`; +- metadata `bridgeProtocolVersion=1.0`. + +Expected output includes the target `invocations_ws` endpoint and wrapper Voice +endpoint. + +## Environment outputs + +```powershell +azd env get-values +``` + +Expected target values: + +```text +AGENT__NAME +AGENT__VERSION +AGENT__PROJECT_ENDPOINT +AGENT__INVOCATIONS_WS_ENDPOINT +``` + +Expected wrapper values: + +```text +AGENT__NAME +AGENT__VERSION +AGENT__PROJECT_ENDPOINT +AGENT__ENDPOINT +AGENT__TARGET_NAME +AGENT__TARGET_VERSION +``` + +The wrapper endpoint is the end-user Voice realtime endpoint. The target +`invocations_ws` endpoint is a diagnostic/developer endpoint. + +## Show and doctor + +```powershell +azd ai agent show voice-target --output json +azd ai agent show voice --output json +azd ai agent doctor --output json +``` + +Expected: + +- target show returns an active hosted definition and `invocations_ws` endpoint; +- wrapper show returns `kind=voice`, `model_type=hosted_agent`, and the pinned + target name/version; +- doctor reports no failed checks. + +`doctor` is project-wide and does not currently accept a service argument. + +## Repeat and independent deployment + +Run the wrapper deployment twice: + +```powershell +azd deploy voice --no-prompt +azd deploy voice --no-prompt +``` + +Expected: the first command creates or updates the wrapper and the second uses +the unified update path. Both preserve a working endpoint. + +The wrapper can be managed independently after its target is deployed: + +```powershell +azd ai agent delete voice --force --no-prompt +azd deploy voice --no-prompt +``` + +Expected: + +- delete removes only the wrapper and clears its environment markers; +- the target remains active; +- the single-service deploy recreates only the wrapper. + +Do not delete the target before its managed wrapper. Full reverse-order +cleanup and ownership-aware `azd down` behavior are planned follow-up work. + +## Direct target protocol smoke + +Use a Voice Bridge Protocol client against: + +```text +AGENT__INVOCATIONS_WS_ENDPOINT +``` + +Expected frame sequence includes `session.ready`, response text events, and +`response.done`. This isolates target code and model access from the Voice +wrapper. It is diagnostic only and is not the end-user path. + +Generic `azd ai agent invoke` intentionally rejects an `invocations_ws`-only +target because the command supports `responses`, `invocations`, and `a2a`. + +## Local Voice dashboard + +From the `voice-first-agent-dev` repository: + +```powershell +cd tests/voice-agent-tests/voice-agents-tests-dashboard/voice_demo +python -m pip install -r requirements.txt +python demo_server.py ` + --backend "/" ` + --bind 127.0.0.1 ` + --port 9527 ` + --auto-delete false +``` + +Open `http://127.0.0.1:9527` on the same machine. + +Confirm: + +1. Agent backend is the intended Foundry project. +2. Inference modes include **Hosted agent**. +3. Agent discovery includes both the hosted target and Voice wrapper. +4. The connection dropdown selects the wrapper, not the target. This is by + design: Voice clients connect to the wrapper. +5. The wrapper details show the expected target name/version. +6. Connect and send a typed turn; text and audio should return. +7. With a microphone available, send a spoken turn and verify STT, target text, + and output audio. + +If the UI reports `getUserMedia ... Requested device not found`, the remote +desktop/browser has no microphone. This does not indicate an agent failure. + +## Negative tests + +Run these against a disposable environment and restore every value after the +test. + +| Case | Expected failure | +|---|---| +| Explicit `AZURE_VOICE_AGENT_API=legacy` | Hosted wrapper requires unified-flat | +| Target version marker points to a missing version | Remote target validation fails before wrapper mutation | +| Target project marker differs from the wrapper project | Dependency validation rejects cross-project binding | +| Target status is not active | Compatibility validation rejects it | +| Target lacks `invocations_ws/1.0.0` | Compatibility validation rejects it | +| Target lacks Voice Bridge metadata | Compatibility validation rejects it | +| Wrapper includes model/instructions/tools/handoff | Manifest validation rejects target-owned fields | +| Wrapper does not list target under `uses` | Target service resolution fails | + +After negative testing, run a normal wrapper deploy and one text turn to verify +the environment was restored. + +## Experience alignment matrix + +| Capability | Hosted/code agent | `invocations_ws` agent | Hosted Voice in this PR | +|---|---|---|---| +| `azure.ai.agent` service | Yes | Yes | Yes, target and wrapper | +| Project reuse/provision | Yes | Yes | Unchanged | +| Package/publish target code | Yes | Yes | Unchanged | +| Service graph deployment | Yes | Yes | Yes, `uses` orders target then wrapper | +| Single-service deploy | Yes | Yes | Yes | +| Remote version pinning | Yes | Yes | Wrapper pins deployed target version | +| `show` | Yes | Yes | Yes for both layers | +| `doctor` | Yes | Yes | Project-wide checks pass | +| Generic `invoke` | Responses/invocations | Not for WS | Not for Voice/WS; use Voice client/UI | +| Delete agent | Yes | Yes | Yes for each layer; wrapper first | +| `azd down` ownership cleanup | Existing behavior | Existing behavior | Follow-up for wrapper reverse cleanup | +| `azd ai agent init` scaffold | Yes | Yes | Follow-up; explicit manifest in this PR | + +## Evidence to record + +For manual sign-off, record: + +- date/time and tester; +- PR commit and installed extension version; +- subscription, region, account, project, and endpoint; +- target and wrapper names/versions; +- output of test/build/package/publish/deploy/show/doctor; +- dashboard screenshot showing Hosted mode and wrapper target binding; +- typed response text; +- spoken input transcript and number/presence of returned audio frames; +- negative test results; +- cleanup status. From 5d5daef9a6397a06a051a38252c878d174dd3358 Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Fri, 21 Aug 2026 13:30:00 +0800 Subject: [PATCH 10/38] fix(agents): address unified voice review feedback --- cli/azd/extensions/azure.ai.agents/README.md | 31 ------------------- .../internal/project/service_target_agent.go | 21 +++++++++---- .../project/service_target_agent_test.go | 27 ++++++++++++++++ 3 files changed, 42 insertions(+), 37 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/README.md b/cli/azd/extensions/azure.ai.agents/README.md index 3108c24e5b7..4ddac596f9a 100644 --- a/cli/azd/extensions/azure.ai.agents/README.md +++ b/cli/azd/extensions/azure.ai.agents/README.md @@ -133,37 +133,6 @@ Details: > the other inline agent properties such as `codeConfiguration` and > `environmentVariables`. -## Prompt voice agent API mode - -Prompt voice agents (`kind: prompt-voice`) use the legacy `/voice_agents` API by -default while the unified `/agents` voice API rolls out across regions. To run -regression tests against the unified API, set `AZURE_VOICE_AGENT_API` before -`azd deploy`: - -```bash -# Default: legacy /voice_agents API -azd env set AZURE_VOICE_AGENT_API legacy - -# Unified /agents API using the current object-shaped audio.output.voice payload -azd env set AZURE_VOICE_AGENT_API unified - -# Unified /agents API using the TiP/spec flat audio.output.voice payload -azd env set AZURE_VOICE_AGENT_API unified-flat -``` - -Details: - -- `legacy` remains the default and preserves existing behavior. -- `unified` and `unified-flat` check `/agents/{name}` remotely before deploying: - `404` creates through `/agents`, while `200` updates through `/agents/{name}`. -- Unified modes write `AGENT__VERSION` and store the callable voice - WebSocket endpoint as `wss://.../agents/{name}/endpoint/protocols/voice?api-version=v1`. -- `legacy` clears any stale `AGENT__VERSION` value and preserves the - existing `/voice_agents/{name}` endpoint marker. -- `unified-flat` is intended for TiP/new-service validation. Non-TiP regions may - still require `legacy` or `unified` until the flat output shape is fully - rolled out. - ### Moderating invocations-protocol traffic For agents that expose the `invocations` protocol, the RAI policy alone is not diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go index a4558598fcb..3b01eb4a811 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go @@ -2330,7 +2330,11 @@ func (p *AgentServiceTargetProvider) deployVoiceAgentWithMode( remoteAgent, getErr := agentClient.GetVoiceAgentUnified( ctx, request.Name, agent_api.AgentEndpointAPIVersion, overriddenHost, ) - if getErr == nil && remoteAgent != nil { + shouldUpdate, decisionErr := shouldUpdateVoiceAgent(remoteAgent, getErr) + if decisionErr != nil { + return nil, exterrors.OpCreateAgent, decisionErr + } + if shouldUpdate { progress("Updating voice agent using unified API") updateRequest := &agent_api.UpdateAgentRequest{ CreateAgentVersionRequest: request.CreateAgentVersionRequest, @@ -2340,17 +2344,22 @@ func (p *AgentServiceTargetProvider) deployVoiceAgentWithMode( ) return agentObject, exterrors.OpUpdateAgent, err } - if getErr != nil { - if respErr, ok := errors.AsType[*azcore.ResponseError](getErr); !ok || respErr.StatusCode != http.StatusNotFound { - return nil, exterrors.OpCreateAgent, getErr - } - } progress("Creating voice agent using unified API") agentObject, err := agentClient.CreateVoiceAgentUnified(ctx, request, agent_api.AgentEndpointAPIVersion, overriddenHost) return agentObject, exterrors.OpCreateAgent, err } +func shouldUpdateVoiceAgent(remoteAgent *agent_api.AgentObject, getErr error) (bool, error) { + if getErr == nil { + return remoteAgent != nil, nil + } + if respErr, ok := errors.AsType[*azcore.ResponseError](getErr); ok && respErr.StatusCode == http.StatusNotFound { + return false, nil + } + return false, getErr +} + func voiceAgentEndpoint(projectEndpoint string, agentName string, apiMode voiceAgentAPIMode) string { trimmedEndpoint := strings.TrimRight(projectEndpoint, "/") if apiMode == voiceAgentAPIModeLegacy { diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go index 69921a0328c..bd4d87fa290 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "net" + "net/http" "os" "path/filepath" "strings" @@ -748,6 +749,32 @@ func TestValidateVoiceAgentDeployResponse(t *testing.T) { }) } +func TestShouldUpdateVoiceAgent(t *testing.T) { + t.Run("remote found updates", func(t *testing.T) { + update, err := shouldUpdateVoiceAgent(&agent_api.AgentObject{Name: "voice"}, nil) + require.NoError(t, err) + require.True(t, update) + }) + + t.Run("remote nil creates", func(t *testing.T) { + update, err := shouldUpdateVoiceAgent(nil, nil) + require.NoError(t, err) + require.False(t, update) + }) + + t.Run("not found creates", func(t *testing.T) { + update, err := shouldUpdateVoiceAgent(nil, &azcore.ResponseError{StatusCode: http.StatusNotFound}) + require.NoError(t, err) + require.False(t, update) + }) + + t.Run("other get error returns error", func(t *testing.T) { + update, err := shouldUpdateVoiceAgent(nil, &azcore.ResponseError{StatusCode: http.StatusInternalServerError}) + require.Error(t, err) + require.False(t, update) + }) +} + func createSymlinkOrSkip(t *testing.T, oldname, newname string) { t.Helper() From bb152c1d63a47f88be9a63f1fe1fe02eb8189da8 Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Thu, 20 Aug 2026 14:14:47 +0800 Subject: [PATCH 11/38] feat(agents): support advanced voice settings --- cli/azd/extensions/azure.ai.agents/README.md | 92 ++++++++ .../extensions/azure.ai.agents/cspell.yaml | 3 + .../internal/pkg/agents/agent_api/models.go | 83 +++++-- .../internal/pkg/agents/agent_yaml/map.go | 216 ++++++++++++++++-- .../pkg/agents/agent_yaml/map_voice_test.go | 102 ++++++++- .../internal/pkg/agents/agent_yaml/parse.go | 69 ++++++ .../internal/pkg/agents/agent_yaml/yaml.go | 89 ++++++++ .../internal/project/agent_definition.go | 67 ++++-- .../internal/project/service_target_agent.go | 21 ++ .../project/service_target_agent_test.go | 8 + .../schemas/azure.ai.agent.json | 159 ++++++++++++- 11 files changed, 836 insertions(+), 73 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/README.md b/cli/azd/extensions/azure.ai.agents/README.md index 4ddac596f9a..8dd303f82a6 100644 --- a/cli/azd/extensions/azure.ai.agents/README.md +++ b/cli/azd/extensions/azure.ai.agents/README.md @@ -199,6 +199,98 @@ keys throughout this block (`invocations_moderation`, `response_mode`, `input_paths`, `stream_selectors`, `event_type`, and so on). The **values** (`non_streaming`, `streaming`, `both`, `json`, `text`) are the same in both. +## Prompt voice advanced configuration + +Advanced prompt voice settings are authored on the `azure.ai.agent` service in +`azure.yaml` and require the unified flat API mode: + +```bash +azd env set AZURE_VOICE_AGENT_API unified-flat +``` + +```yaml +services: + voice-agent: + host: azure.ai.agent + kind: prompt-voice + name: voice-agent + modelType: managed # or self_deployed for BYOM + model: + id: gpt-realtime + instructions: You are {{persona}}, a concise voice assistant. + structuredInputs: + persona: + description: Assistant persona + defaultValue: Ada + schema: + type: string + audio: + input: + format: + type: audio/pcmu + noiseReduction: + type: near_field + echoCancellation: + type: server_echo_cancellation + reference_source: server + channels: 1 + turnDetection: + type: azure_semantic_vad + threshold: 0.6 + speechDurationMs: 120 + removeFillerWords: true + createResponse: true + interruptResponse: true + languages: [en-US] + autoTruncate: true + transcription: + model: whisper-1 + language: en-US + output: + format: + type: audio/pcm + rate: 24000 + voice: + type: azure_standard + name: en-US-AvaNeural + locale: en-US + style: cheerful + speed: 1.0 + outputModalities: [audio, text] + tools: + - type: system + name: end_conversation + avatar: + type: video_avatar + character: lisa + style: casual-sitting + output_protocol: webrtc + greeting: + type: template + text: Hello {{persona}} + toolChoice: auto + parallelToolCalls: true + maxOutputTokens: inf + include: + - item.input_audio_transcription.phrases +``` + +Notes: + +- `voice`, `instructions`, and `store` remain supported for simple prompt voice + agents. Missing audio fields keep the existing azd defaults. +- `audio.output.voice` uses an author-friendly object shape; in `unified-flat` + mode azd maps it to the service flat fields `voice`, `voice_type`, + `voice_locale`, `style`, `pitch`, `rate`, and `volume`. +- `structuredInputs.defaultValue` maps to the service wire field + `default_value`. +- Direct voice tool types are `function`, `mcp`, `system`, and `toolbox`. + Server-side tools such as `web_search`, `azure_ai_search`, and `openapi` must + be packaged through a toolbox. +- `tools`, `avatar`, `greeting`, `handoff`, `toolChoice`, and + `echoCancellation` intentionally remain light pass-through blocks so azd does + not block new service-side additions. + ## Session idle timeout A hosted agent's runtime session sandbox is suspended by Foundry after a period diff --git a/cli/azd/extensions/azure.ai.agents/cspell.yaml b/cli/azd/extensions/azure.ai.agents/cspell.yaml index 5d2d9e0950c..78be91d2fea 100644 --- a/cli/azd/extensions/azure.ai.agents/cspell.yaml +++ b/cli/azd/extensions/azure.ai.agents/cspell.yaml @@ -11,6 +11,9 @@ words: # Voice (prompt-voice) agents - BYOM - Nanami + - pcma + - pcmu + - webrtc # Azure region names - australiaeast - brazilsouth diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go index 9fedb6e56e3..ec2ea32be40 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go @@ -360,7 +360,7 @@ const ( // VoiceAudioFormat describes a PCM audio stream format (e.g. audio/pcm @ 24 kHz). type VoiceAudioFormat struct { Type string `json:"type"` - Rate int `json:"rate"` + Rate *int `json:"rate,omitempty"` } // VoiceTurnDetection configures server-side voice-activity detection so the @@ -370,32 +370,54 @@ type VoiceTurnDetection struct { Threshold *float64 `json:"threshold,omitempty"` PrefixPaddingMs *int `json:"prefix_padding_ms,omitempty"` SilenceDurationMs *int `json:"silence_duration_ms,omitempty"` + CreateResponse *bool `json:"create_response,omitempty"` + Eagerness *string `json:"eagerness,omitempty"` + SpeechDurationMs *int `json:"speech_duration_ms,omitempty"` + RemoveFillerWords *bool `json:"remove_filler_words,omitempty"` + InterruptResponse *bool `json:"interrupt_response,omitempty"` + Languages []string `json:"languages,omitempty"` + AutoTruncate *bool `json:"auto_truncate,omitempty"` } // VoiceTranscription enables user-speech transcription events on the input stream. type VoiceTranscription struct { - Model string `json:"model,omitempty"` + Model string `json:"model,omitempty"` + Language *string `json:"language,omitempty"` + Prompt *string `json:"prompt,omitempty"` +} + +// VoiceNoiseReduction configures input audio noise reduction. +type VoiceNoiseReduction struct { + Type string `json:"type"` } // VoiceInputConfig is the input (caller -> agent) audio configuration. type VoiceInputConfig struct { - Format *VoiceAudioFormat `json:"format,omitempty"` - TurnDetection *VoiceTurnDetection `json:"turn_detection,omitempty"` - Transcription *VoiceTranscription `json:"transcription,omitempty"` + Format *VoiceAudioFormat `json:"format,omitempty"` + NoiseReduction *VoiceNoiseReduction `json:"noise_reduction,omitempty"` + EchoCancellation map[string]any `json:"echo_cancellation,omitempty"` + TurnDetection *VoiceTurnDetection `json:"turn_detection,omitempty"` + Transcription *VoiceTranscription `json:"transcription,omitempty"` } // VoiceConfig selects the output voice. Type is "openai" for realtime voices // (single lowercase word, e.g. "alloy") or "azure_standard" for Azure Neural // voices (e.g. "en-US-Ava:DragonHDLatestNeural"). type VoiceConfig struct { - Type string `json:"type"` - Name string `json:"name"` + Type string `json:"type"` + Name string `json:"name"` + Style *string `json:"style,omitempty"` + Pitch *string `json:"pitch,omitempty"` + Rate *string `json:"rate,omitempty"` + Locale *string `json:"locale,omitempty"` + Volume *string `json:"volume,omitempty"` } // VoiceOutputConfig is the output (agent -> caller) audio configuration. type VoiceOutputConfig struct { Format *VoiceAudioFormat `json:"format,omitempty"` Voice *VoiceConfig `json:"voice,omitempty"` + Speed *float64 `json:"speed,omitempty"` } // VoiceOutputConfigFlat is the newer Voice Live output shape used by the @@ -406,6 +428,11 @@ type VoiceOutputConfigFlat struct { Voice string `json:"voice,omitempty"` VoiceType string `json:"voice_type,omitempty"` VoiceLocale string `json:"voice_locale,omitempty"` + Style *string `json:"style,omitempty"` + Pitch *string `json:"pitch,omitempty"` + Rate *string `json:"rate,omitempty"` + Volume *string `json:"volume,omitempty"` + Speed *float64 `json:"speed,omitempty"` } // VoiceAudioConfig bundles the input and output audio configuration. @@ -425,12 +452,21 @@ type VoiceAudioConfigFlat struct { // is always AgentKindVoice ("voice"). type VoiceAgentDefinition struct { AgentDefinition - ModelType VoiceModelType `json:"model_type"` - Model string `json:"model"` - Instructions string `json:"instructions,omitempty"` - Audio *VoiceAudioConfig `json:"audio,omitempty"` - OutputModalities []string `json:"output_modalities,omitempty"` - Store *bool `json:"store,omitempty"` + ModelType VoiceModelType `json:"model_type"` + Model string `json:"model"` + Instructions string `json:"instructions,omitempty"` + StructuredInputs map[string]any `json:"structured_inputs,omitempty"` + Audio *VoiceAudioConfig `json:"audio,omitempty"` + OutputModalities []string `json:"output_modalities,omitempty"` + Store *bool `json:"store,omitempty"` + Tools []map[string]any `json:"tools,omitempty"` + Avatar map[string]any `json:"avatar,omitempty"` + Greeting map[string]any `json:"greeting,omitempty"` + Handoff map[string]any `json:"handoff,omitempty"` + ToolChoice any `json:"tool_choice,omitempty"` + ParallelToolCalls *bool `json:"parallel_tool_calls,omitempty"` + MaxOutputTokens any `json:"max_output_tokens,omitempty"` + Include []string `json:"include,omitempty"` } // VoiceAgentDefinitionFlat is the voice definition shape aligned with the @@ -438,12 +474,21 @@ type VoiceAgentDefinition struct { // provider details are sibling fields. type VoiceAgentDefinitionFlat struct { AgentDefinition - ModelType VoiceModelType `json:"model_type"` - Model string `json:"model"` - Instructions string `json:"instructions,omitempty"` - Audio *VoiceAudioConfigFlat `json:"audio,omitempty"` - OutputModalities []string `json:"output_modalities,omitempty"` - Store *bool `json:"store,omitempty"` + ModelType VoiceModelType `json:"model_type"` + Model string `json:"model"` + Instructions string `json:"instructions,omitempty"` + StructuredInputs map[string]any `json:"structured_inputs,omitempty"` + Audio *VoiceAudioConfigFlat `json:"audio,omitempty"` + OutputModalities []string `json:"output_modalities,omitempty"` + Store *bool `json:"store,omitempty"` + Tools []map[string]any `json:"tools,omitempty"` + Avatar map[string]any `json:"avatar,omitempty"` + Greeting map[string]any `json:"greeting,omitempty"` + Handoff map[string]any `json:"handoff,omitempty"` + ToolChoice any `json:"tool_choice,omitempty"` + ParallelToolCalls *bool `json:"parallel_tool_calls,omitempty"` + MaxOutputTokens any `json:"max_output_tokens,omitempty"` + Include []string `json:"include,omitempty"` } // CreateAgentVersionRequest represents a request to create an agent version diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go index 42b1463fb05..0ea5fe41d89 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go @@ -593,10 +593,22 @@ func flatVoiceType(voice *agent_api.VoiceConfig) string { return voice.Type } +func normalizeFlatVoiceType(voiceType string) string { + switch strings.TrimSpace(voiceType) { + case "azure_standard": + return "azure-standard" + default: + return strings.TrimSpace(voiceType) + } +} + func flatVoiceLocale(voice *agent_api.VoiceConfig) string { if voice == nil || voice.Name == "" || isOpenAIVoice(voice.Name) { return "" } + if voice.Locale != nil && strings.TrimSpace(*voice.Locale) != "" { + return strings.TrimSpace(*voice.Locale) + } parts := strings.SplitN(voice.Name, "-", 3) if len(parts) < 2 { return "" @@ -604,6 +616,114 @@ func flatVoiceLocale(voice *agent_api.VoiceConfig) string { return parts[0] + "-" + parts[1] } +func defaultVoiceAudioFormat() *agent_api.VoiceAudioFormat { + rate := defaultVoiceAudioRate + return &agent_api.VoiceAudioFormat{Type: defaultVoiceAudioType, Rate: &rate} +} + +func mapVoiceAudioFormat(format *VoiceAudioFormat, fallback *agent_api.VoiceAudioFormat) *agent_api.VoiceAudioFormat { + out := &agent_api.VoiceAudioFormat{} + if fallback != nil { + *out = *fallback + } + if format != nil { + if strings.TrimSpace(format.Type) != "" { + out.Type = strings.TrimSpace(format.Type) + } + if format.Rate != nil { + out.Rate = format.Rate + } + } + return out +} + +func mapVoiceTurnDetection(turnDetection *VoiceTurnDetection) *agent_api.VoiceTurnDetection { + out := &agent_api.VoiceTurnDetection{Type: defaultVoiceTurnDetectionType} + if turnDetection == nil { + return out + } + if strings.TrimSpace(turnDetection.Type) != "" { + out.Type = strings.TrimSpace(turnDetection.Type) + } + out.Threshold = turnDetection.Threshold + out.PrefixPaddingMs = turnDetection.PrefixPaddingMs + out.SilenceDurationMs = turnDetection.SilenceDurationMs + out.CreateResponse = turnDetection.CreateResponse + out.Eagerness = turnDetection.Eagerness + out.SpeechDurationMs = turnDetection.SpeechDurationMs + out.RemoveFillerWords = turnDetection.RemoveFillerWords + out.InterruptResponse = turnDetection.InterruptResponse + out.Languages = turnDetection.Languages + out.AutoTruncate = turnDetection.AutoTruncate + return out +} + +func mapVoiceTranscription(transcription *VoiceTranscription) *agent_api.VoiceTranscription { + out := &agent_api.VoiceTranscription{Model: defaultVoiceInputTranscriptionModel} + if transcription == nil { + return out + } + if strings.TrimSpace(transcription.Model) != "" { + out.Model = strings.TrimSpace(transcription.Model) + } + out.Language = transcription.Language + out.Prompt = transcription.Prompt + return out +} + +func mapVoiceConfig(voice *VoiceConfig, fallbackName string) *agent_api.VoiceConfig { + if voice == nil { + return buildVoiceConfig(fallbackName) + } + name := strings.TrimSpace(voice.Name) + if name == "" { + name = fallbackName + } + voiceType := strings.TrimSpace(voice.Type) + if voiceType == "" { + out := buildVoiceConfig(name) + out.Style = voice.Style + out.Pitch = voice.Pitch + out.Rate = voice.Rate + out.Locale = voice.Locale + out.Volume = voice.Volume + return out + } + return &agent_api.VoiceConfig{ + Type: voiceType, + Name: name, + Style: voice.Style, + Pitch: voice.Pitch, + Rate: voice.Rate, + Locale: voice.Locale, + Volume: voice.Volume, + } +} + +func mapVoiceStructuredInputs(inputs map[string]any) map[string]any { + if len(inputs) == 0 { + return nil + } + out := make(map[string]any, len(inputs)) + for name, input := range inputs { + inputMap, ok := input.(map[string]any) + if !ok { + out[name] = input + continue + } + + mapped := maps.Clone(inputMap) + if value, ok := mapped["defaultValue"]; ok { + if _, hasSnakeCase := mapped["default_value"]; !hasSnakeCase { + mapped["default_value"] = value + } + delete(mapped, "defaultValue") + } + out[name] = mapped + } + return out +} + // CreateVoiceAgentAPIRequest builds a CreateAgentRequest for a declarative // voice agent. It translates the authoring kind "prompt-voice" into the // data-plane service kind "voice" and defaults the audio pipeline. @@ -646,37 +766,77 @@ func createVoiceAgentAPIRequest(voiceAgent VoiceAgent, flatOutput bool) (*agent_ voiceName = *voiceAgent.Voice } - audioFormat := &agent_api.VoiceAudioFormat{ - Type: defaultVoiceAudioType, - Rate: defaultVoiceAudioRate, + inputFormat := defaultVoiceAudioFormat() + outputFormat := defaultVoiceAudioFormat() + turnDetection := mapVoiceTurnDetection(nil) + transcription := mapVoiceTranscription(nil) + var noiseReduction *agent_api.VoiceNoiseReduction + var echoCancellation map[string]any + outputVoice := buildVoiceConfig(voiceName) + var outputSpeed *float64 + if voiceAgent.Audio != nil { + if voiceAgent.Audio.Input != nil { + inputFormat = mapVoiceAudioFormat(voiceAgent.Audio.Input.Format, inputFormat) + if voiceAgent.Audio.Input.NoiseReduction != nil { + noiseReduction = &agent_api.VoiceNoiseReduction{Type: strings.TrimSpace(voiceAgent.Audio.Input.NoiseReduction.Type)} + } + echoCancellation = voiceAgent.Audio.Input.EchoCancellation + turnDetection = mapVoiceTurnDetection(voiceAgent.Audio.Input.TurnDetection) + transcription = mapVoiceTranscription(voiceAgent.Audio.Input.Transcription) + } + if voiceAgent.Audio.Output != nil { + outputFormat = mapVoiceAudioFormat(voiceAgent.Audio.Output.Format, outputFormat) + outputVoice = mapVoiceConfig(voiceAgent.Audio.Output.Voice, voiceName) + outputSpeed = voiceAgent.Audio.Output.Speed + } + } + + outputModalities := []string{"audio"} + if len(voiceAgent.OutputModalities) > 0 { + outputModalities = voiceAgent.OutputModalities } input := &agent_api.VoiceInputConfig{ - Format: audioFormat, - TurnDetection: &agent_api.VoiceTurnDetection{Type: defaultVoiceTurnDetectionType}, - Transcription: &agent_api.VoiceTranscription{Model: defaultVoiceInputTranscriptionModel}, + Format: inputFormat, + NoiseReduction: noiseReduction, + EchoCancellation: echoCancellation, + TurnDetection: turnDetection, + Transcription: transcription, } - voiceConfig := buildVoiceConfig(voiceName) if flatOutput { voiceDef := agent_api.VoiceAgentDefinitionFlat{ AgentDefinition: agent_api.AgentDefinition{ // Translate authoring kind prompt-voice -> service kind voice. Kind: agent_api.AgentKindVoice, }, - ModelType: modelType, - Model: modelID, - Instructions: instructions, + ModelType: modelType, + Model: modelID, + Instructions: instructions, + StructuredInputs: mapVoiceStructuredInputs(voiceAgent.StructuredInputs), Audio: &agent_api.VoiceAudioConfigFlat{ Input: input, Output: &agent_api.VoiceOutputConfigFlat{ - Format: audioFormat, - Voice: voiceConfig.Name, - VoiceType: flatVoiceType(voiceConfig), - VoiceLocale: flatVoiceLocale(voiceConfig), + Format: outputFormat, + Voice: outputVoice.Name, + VoiceType: normalizeFlatVoiceType(flatVoiceType(outputVoice)), + VoiceLocale: flatVoiceLocale(outputVoice), + Style: outputVoice.Style, + Pitch: outputVoice.Pitch, + Rate: outputVoice.Rate, + Volume: outputVoice.Volume, + Speed: outputSpeed, }, }, - OutputModalities: []string{"audio"}, - Store: voiceAgent.Store, + OutputModalities: outputModalities, + Store: voiceAgent.Store, + Tools: voiceAgent.Tools, + Avatar: voiceAgent.Avatar, + Greeting: voiceAgent.Greeting, + Handoff: voiceAgent.Handoff, + ToolChoice: voiceAgent.ToolChoice, + ParallelToolCalls: voiceAgent.ParallelToolCalls, + MaxOutputTokens: voiceAgent.MaxOutputTokens, + Include: voiceAgent.Include, } return createAgentAPIRequest(voiceAgent.AgentDefinition, voiceDef, nil, nil) @@ -687,18 +847,28 @@ func createVoiceAgentAPIRequest(voiceAgent VoiceAgent, flatOutput bool) (*agent_ // Translate authoring kind prompt-voice -> service kind voice. Kind: agent_api.AgentKindVoice, }, - ModelType: modelType, - Model: modelID, - Instructions: instructions, + ModelType: modelType, + Model: modelID, + Instructions: instructions, + StructuredInputs: mapVoiceStructuredInputs(voiceAgent.StructuredInputs), Audio: &agent_api.VoiceAudioConfig{ Input: input, Output: &agent_api.VoiceOutputConfig{ - Format: audioFormat, - Voice: voiceConfig, + Format: outputFormat, + Voice: outputVoice, + Speed: outputSpeed, }, }, - OutputModalities: []string{"audio"}, - Store: voiceAgent.Store, + OutputModalities: outputModalities, + Store: voiceAgent.Store, + Tools: voiceAgent.Tools, + Avatar: voiceAgent.Avatar, + Greeting: voiceAgent.Greeting, + Handoff: voiceAgent.Handoff, + ToolChoice: voiceAgent.ToolChoice, + ParallelToolCalls: voiceAgent.ParallelToolCalls, + MaxOutputTokens: voiceAgent.MaxOutputTokens, + Include: voiceAgent.Include, } return createAgentAPIRequest(voiceAgent.AgentDefinition, voiceDef, nil, nil) diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go index 1e536b344da..115538c572b 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go @@ -10,6 +10,8 @@ import ( "azureaiagent/internal/pkg/agents/agent_api" ) +func ptr[T any](v T) *T { return &v } + // --------------------------------------------------------------------------- // isOpenAIVoice / buildVoiceConfig // --------------------------------------------------------------------------- @@ -118,7 +120,8 @@ func TestCreateVoiceAgentAPIRequest_Defaults(t *testing.T) { t.Fatalf("Audio pipeline not populated: %+v", def.Audio) } in := def.Audio.Input - if in.Format == nil || in.Format.Type != defaultVoiceAudioType || in.Format.Rate != defaultVoiceAudioRate { + if in.Format == nil || in.Format.Type != defaultVoiceAudioType || in.Format.Rate == nil || + *in.Format.Rate != defaultVoiceAudioRate { t.Errorf("input format = %+v", in.Format) } if in.TurnDetection == nil || in.TurnDetection.Type != defaultVoiceTurnDetectionType { @@ -128,7 +131,8 @@ func TestCreateVoiceAgentAPIRequest_Defaults(t *testing.T) { t.Errorf("transcription = %+v", in.Transcription) } out := def.Audio.Output - if out.Format == nil || out.Format.Type != defaultVoiceAudioType || out.Format.Rate != defaultVoiceAudioRate { + if out.Format == nil || out.Format.Type != defaultVoiceAudioType || out.Format.Rate == nil || + *out.Format.Rate != defaultVoiceAudioRate { t.Errorf("output format = %+v", out.Format) } // Default voice is the DragonHD Azure Neural voice. @@ -263,6 +267,100 @@ func TestCreateVoiceAgentAPIRequestFlat_MarshalWireShape(t *testing.T) { } } +func TestCreateVoiceAgentAPIRequestFlat_AdvancedSettingsWireShape(t *testing.T) { + t.Parallel() + inRate := 16000 + outRate := 24000 + threshold := 0.6 + speechDurationMs := 120 + createResponse := true + removeFillerWords := true + interruptResponse := true + autoTruncate := true + speed := 1.1 + parallelToolCalls := true + style := "cheerful" + pitch := "+0Hz" + rate := "+0%" + volume := "+0%" + agent := VoiceAgent{ + AgentDefinition: AgentDefinition{Kind: AgentKindPromptVoice, Name: "voice-advanced"}, + Model: &Model{Id: "gpt-realtime"}, + Instructions: ptr("You are {{persona}}, a concise voice assistant."), + StructuredInputs: map[string]any{ + "persona": map[string]any{"description": "Assistant persona", "defaultValue": "Ada"}, + }, + Audio: &VoiceAudio{ + Input: &VoiceAudioInput{ + Format: &VoiceAudioFormat{Type: "audio/pcmu", Rate: &inRate}, + NoiseReduction: &VoiceNoiseReduction{Type: "near_field"}, + EchoCancellation: map[string]any{"type": "server_echo_cancellation", "channels": 1}, + TurnDetection: &VoiceTurnDetection{ + Type: "azure_semantic_vad", + Threshold: &threshold, + SpeechDurationMs: &speechDurationMs, + CreateResponse: &createResponse, + RemoveFillerWords: &removeFillerWords, + InterruptResponse: &interruptResponse, + Languages: []string{"en-US"}, + AutoTruncate: &autoTruncate, + }, + Transcription: &VoiceTranscription{Model: "whisper-1", Language: ptr("en-US"), Prompt: ptr("Contoso terms")}, + }, + Output: &VoiceAudioOutput{ + Format: &VoiceAudioFormat{Type: "audio/pcm", Rate: &outRate}, + Voice: &VoiceConfig{ + Type: "azure_standard", Name: "en-US-AvaNeural", Style: &style, + Pitch: &pitch, Rate: &rate, Locale: ptr("en-US"), Volume: &volume, + }, + Speed: &speed, + }, + }, + OutputModalities: []string{"audio", "text"}, + Tools: []map[string]any{{"type": "system", "name": "end_conversation"}}, + Avatar: map[string]any{"type": "video_avatar", "character": "lisa", "output_protocol": "webrtc"}, + Greeting: map[string]any{"type": "template", "text": "Hello {{persona}}"}, + ToolChoice: "auto", + ParallelToolCalls: ¶llelToolCalls, + MaxOutputTokens: "inf", + Include: []string{"item.input_audio_transcription.phrases"}, + } + + req, err := CreateVoiceAgentAPIRequestFlat(agent) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + payload, err := json.Marshal(req) + if err != nil { + t.Fatalf("marshal request: %v", err) + } + + var wire map[string]any + if err := json.Unmarshal(payload, &wire); err != nil { + t.Fatalf("unmarshal payload: %v", err) + } + def := wire["definition"].(map[string]any) + input := def["audio"].(map[string]any)["input"].(map[string]any) + output := def["audio"].(map[string]any)["output"].(map[string]any) + structured := def["structured_inputs"].(map[string]any)["persona"].(map[string]any) + + if structured["default_value"] != "Ada" || structured["defaultValue"] != nil { + t.Fatalf("structured input default was not mapped to wire shape: %#v", structured) + } + if output["voice"] != "en-US-AvaNeural" || output["voice_type"] != "azure-standard" || output["style"] != style { + t.Fatalf("output voice flat shape not mapped: %#v", output) + } + if input["echo_cancellation"].(map[string]any)["type"] != "server_echo_cancellation" { + t.Fatalf("echo cancellation not mapped: %#v", input["echo_cancellation"]) + } + if def["tool_choice"] != "auto" || def["max_output_tokens"] != "inf" { + t.Fatalf("response options not mapped: %#v", def) + } + if len(def["tools"].([]any)) != 1 || def["avatar"].(map[string]any)["character"] != "lisa" { + t.Fatalf("tools/avatar not mapped: %#v", def) + } +} + // TestCreateVoiceAgentAPIRequest_ExplicitManaged verifies that explicitly // setting model_type: managed is accepted (idempotent with the default). func TestCreateVoiceAgentAPIRequest_ExplicitManaged(t *testing.T) { diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go index 996eae8a561..42e894667fc 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go @@ -460,6 +460,7 @@ func ValidateAgentDefinition(templateBytes []byte) error { "template.model_type '%s' is not supported; use '%s' or '%s'", agent.ModelType, VoiceModelTypeManaged, VoiceModelTypeSelfDeployed)) } + errors = append(errors, validateVoiceAgentAdvancedConfig(agent)...) } else { errors = append(errors, fmt.Sprintf("failed to unmarshal to VoiceAgent: %v", err)) } @@ -479,6 +480,74 @@ func ValidateAgentDefinition(templateBytes []byte) error { return nil } +func validateVoiceAgentAdvancedConfig(agent VoiceAgent) []string { + var errors []string + for i, modality := range agent.OutputModalities { + if strings.TrimSpace(modality) == "" { + errors = append(errors, fmt.Sprintf("template.output_modalities[%d] must not be blank", i)) + } + } + + if agent.Audio == nil { + return errors + } + if agent.Audio.Input != nil { + errors = append(errors, validateVoiceAudioFormat("template.audio.input.format", agent.Audio.Input.Format)...) + if nr := agent.Audio.Input.NoiseReduction; nr != nil && strings.TrimSpace(nr.Type) == "" { + errors = append(errors, "template.audio.input.noise_reduction.type must not be blank") + } + if td := agent.Audio.Input.TurnDetection; td != nil { + if strings.TrimSpace(td.Type) == "" { + errors = append(errors, "template.audio.input.turn_detection.type must not be blank") + } + if td.Threshold != nil && (*td.Threshold < 0 || *td.Threshold > 1) { + errors = append(errors, "template.audio.input.turn_detection.threshold must be between 0 and 1") + } + if td.PrefixPaddingMs != nil && *td.PrefixPaddingMs < 0 { + errors = append(errors, "template.audio.input.turn_detection.prefix_padding_ms must be >= 0") + } + if td.SilenceDurationMs != nil && *td.SilenceDurationMs < 0 { + errors = append(errors, "template.audio.input.turn_detection.silence_duration_ms must be >= 0") + } + if td.SpeechDurationMs != nil && *td.SpeechDurationMs < 0 { + errors = append(errors, "template.audio.input.turn_detection.speech_duration_ms must be >= 0") + } + } + } + if agent.Audio.Output != nil { + errors = append(errors, validateVoiceAudioFormat("template.audio.output.format", agent.Audio.Output.Format)...) + if voice := agent.Audio.Output.Voice; voice != nil { + if strings.TrimSpace(voice.Type) == "" { + errors = append(errors, "template.audio.output.voice.type must not be blank") + } + if strings.TrimSpace(voice.Name) == "" { + errors = append(errors, "template.audio.output.voice.name must not be blank") + } + } + if speed := agent.Audio.Output.Speed; speed != nil && (*speed < 0.25 || *speed > 1.5) { + errors = append(errors, "template.audio.output.speed must be between 0.25 and 1.5") + } + } + return errors +} + +func validateVoiceAudioFormat(path string, format *VoiceAudioFormat) []string { + if format == nil { + return nil + } + var errors []string + formatType := strings.TrimSpace(format.Type) + if formatType == "" { + errors = append(errors, path+".type must not be blank") + } else if formatType != "audio/pcm" && formatType != "audio/pcmu" && formatType != "audio/pcma" { + errors = append(errors, path+".type must be 'audio/pcm', 'audio/pcmu', or 'audio/pcma'") + } + if format.Rate != nil && *format.Rate <= 0 { + errors = append(errors, path+".rate must be greater than 0") + } + return errors +} + // Validate that the agent name matches the expected deployable format func ValidateAgentName(name string) error { if name == "" { diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/yaml.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/yaml.go index 46fffca9e6a..ffa35d057db 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/yaml.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/yaml.go @@ -214,9 +214,98 @@ type VoiceAgent struct { // Voice is the output voice name (e.g. "en-US-Ava:DragonHDLatestNeural" for // an Azure Neural voice, or "alloy" for an OpenAI realtime voice). Voice *string `json:"voice,omitempty" yaml:"voice,omitempty"` + // StructuredInputs declares template inputs used by voice instructions and greeting. + StructuredInputs map[string]any `json:"structuredInputs,omitempty" yaml:"structured_inputs,omitempty"` + // Audio customizes the input and output voice pipeline. Missing fields keep azd defaults. + Audio *VoiceAudio `json:"audio,omitempty" yaml:"audio,omitempty"` + // OutputModalities declares response modalities such as audio, text, animation, or avatar. + OutputModalities []string `json:"outputModalities,omitempty" yaml:"output_modalities,omitempty"` // Store toggles server-side logging (transcript + per-turn audio). Optional; // the service defaults to false when omitted. Store *bool `json:"store,omitempty" yaml:"store,omitempty"` + // Tools are passed through to the prompt voice service. Supported direct tool + // types include function, mcp, system, and toolbox. + Tools []map[string]any `json:"tools,omitempty" yaml:"tools,omitempty"` + // Avatar customizes voice avatar output for services that support it. + Avatar map[string]any `json:"avatar,omitempty" yaml:"avatar,omitempty"` + // Greeting configures initial greeting behavior for services that support it. + Greeting map[string]any `json:"greeting,omitempty" yaml:"greeting,omitempty"` + // Handoff configures voice handoff behavior for services that support it. + Handoff map[string]any `json:"handoff,omitempty" yaml:"handoff,omitempty"` + // ToolChoice configures service tool choice behavior, such as auto/none/required. + ToolChoice any `json:"toolChoice,omitempty" yaml:"tool_choice,omitempty"` + // ParallelToolCalls toggles parallel tool calls. + ParallelToolCalls *bool `json:"parallelToolCalls,omitempty" yaml:"parallel_tool_calls,omitempty"` + // MaxOutputTokens limits response output tokens. Use an integer or service-supported string such as "inf". + MaxOutputTokens any `json:"maxOutputTokens,omitempty" yaml:"max_output_tokens,omitempty"` + // Include requests additional service response fields. + Include []string `json:"include,omitempty" yaml:"include,omitempty"` +} + +// VoiceAudio bundles optional prompt voice input/output audio overrides. +type VoiceAudio struct { + Input *VoiceAudioInput `json:"input,omitempty" yaml:"input,omitempty"` + Output *VoiceAudioOutput `json:"output,omitempty" yaml:"output,omitempty"` +} + +// VoiceAudioInput customizes caller-to-agent audio. +type VoiceAudioInput struct { + Format *VoiceAudioFormat `json:"format,omitempty" yaml:"format,omitempty"` + NoiseReduction *VoiceNoiseReduction `json:"noiseReduction,omitempty" yaml:"noise_reduction,omitempty"` + EchoCancellation map[string]any `json:"echoCancellation,omitempty" yaml:"echo_cancellation,omitempty"` + TurnDetection *VoiceTurnDetection `json:"turnDetection,omitempty" yaml:"turn_detection,omitempty"` + Transcription *VoiceTranscription `json:"transcription,omitempty" yaml:"transcription,omitempty"` +} + +// VoiceAudioOutput customizes agent-to-caller audio. +type VoiceAudioOutput struct { + Format *VoiceAudioFormat `json:"format,omitempty" yaml:"format,omitempty"` + Voice *VoiceConfig `json:"voice,omitempty" yaml:"voice,omitempty"` + Speed *float64 `json:"speed,omitempty" yaml:"speed,omitempty"` +} + +// VoiceAudioFormat describes an audio stream format. +type VoiceAudioFormat struct { + Type string `json:"type" yaml:"type"` + Rate *int `json:"rate,omitempty" yaml:"rate,omitempty"` +} + +// VoiceNoiseReduction configures input audio noise reduction. +type VoiceNoiseReduction struct { + Type string `json:"type" yaml:"type"` +} + +// VoiceTurnDetection configures server-side turn detection. +type VoiceTurnDetection struct { + Type string `json:"type" yaml:"type"` + Threshold *float64 `json:"threshold,omitempty" yaml:"threshold,omitempty"` + PrefixPaddingMs *int `json:"prefixPaddingMs,omitempty" yaml:"prefix_padding_ms,omitempty"` + SilenceDurationMs *int `json:"silenceDurationMs,omitempty" yaml:"silence_duration_ms,omitempty"` + CreateResponse *bool `json:"createResponse,omitempty" yaml:"create_response,omitempty"` + Eagerness *string `json:"eagerness,omitempty" yaml:"eagerness,omitempty"` + SpeechDurationMs *int `json:"speechDurationMs,omitempty" yaml:"speech_duration_ms,omitempty"` + RemoveFillerWords *bool `json:"removeFillerWords,omitempty" yaml:"remove_filler_words,omitempty"` + InterruptResponse *bool `json:"interruptResponse,omitempty" yaml:"interrupt_response,omitempty"` + Languages []string `json:"languages,omitempty" yaml:"languages,omitempty"` + AutoTruncate *bool `json:"autoTruncate,omitempty" yaml:"auto_truncate,omitempty"` +} + +// VoiceTranscription configures input transcription. +type VoiceTranscription struct { + Model string `json:"model,omitempty" yaml:"model,omitempty"` + Language *string `json:"language,omitempty" yaml:"language,omitempty"` + Prompt *string `json:"prompt,omitempty" yaml:"prompt,omitempty"` +} + +// VoiceConfig selects the output voice. +type VoiceConfig struct { + Type string `json:"type" yaml:"type"` + Name string `json:"name" yaml:"name"` + Style *string `json:"style,omitempty" yaml:"style,omitempty"` + Pitch *string `json:"pitch,omitempty" yaml:"pitch,omitempty"` + Rate *string `json:"rate,omitempty" yaml:"rate,omitempty"` + Locale *string `json:"locale,omitempty" yaml:"locale,omitempty"` + Volume *string `json:"volume,omitempty" yaml:"volume,omitempty"` } // ContainerResources represents the resource allocation for a containerized agent. diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/agent_definition.go b/cli/azd/extensions/azure.ai.agents/internal/project/agent_definition.go index 26b5a06f874..1625d9a7637 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/agent_definition.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/agent_definition.go @@ -138,35 +138,68 @@ type AgentDefinitionInline struct { // Voice-agent fields (kind: prompt-voice). All omitempty so container/ // workflow entries are byte-for-byte unchanged. - ModelType agent_yaml.VoiceModelType `json:"modelType,omitempty"` - Model *agent_yaml.Model `json:"model,omitempty"` - Instructions *string `json:"instructions,omitempty"` - Voice *string `json:"voice,omitempty"` - Store *bool `json:"store,omitempty"` + ModelType agent_yaml.VoiceModelType `json:"modelType,omitempty"` + Model *agent_yaml.Model `json:"model,omitempty"` + Instructions *string `json:"instructions,omitempty"` + Voice *string `json:"voice,omitempty"` + StructuredInputs map[string]any `json:"structuredInputs,omitempty"` + Audio *agent_yaml.VoiceAudio `json:"audio,omitempty"` + OutputModalities []string `json:"outputModalities,omitempty"` + Store *bool `json:"store,omitempty"` + Tools []map[string]any `json:"tools,omitempty"` + Avatar map[string]any `json:"avatar,omitempty"` + Greeting map[string]any `json:"greeting,omitempty"` + Handoff map[string]any `json:"handoff,omitempty"` + ToolChoice any `json:"toolChoice,omitempty"` + ParallelToolCalls *bool `json:"parallelToolCalls,omitempty"` + MaxOutputTokens any `json:"maxOutputTokens,omitempty"` + Include []string `json:"include,omitempty"` } // voiceAgentDefinitionToInline projects a VoiceAgent into the inline definition // written to azure.yaml. Voice agents carry no container/image/code config. func voiceAgentDefinitionToInline(va agent_yaml.VoiceAgent) AgentDefinitionInline { return AgentDefinitionInline{ - AgentDefinition: va.AgentDefinition, - ModelType: va.ModelType, - Model: va.Model, - Instructions: va.Instructions, - Voice: va.Voice, - Store: va.Store, + AgentDefinition: va.AgentDefinition, + ModelType: va.ModelType, + Model: va.Model, + Instructions: va.Instructions, + Voice: va.Voice, + StructuredInputs: va.StructuredInputs, + Audio: va.Audio, + OutputModalities: va.OutputModalities, + Store: va.Store, + Tools: va.Tools, + Avatar: va.Avatar, + Greeting: va.Greeting, + Handoff: va.Handoff, + ToolChoice: va.ToolChoice, + ParallelToolCalls: va.ParallelToolCalls, + MaxOutputTokens: va.MaxOutputTokens, + Include: va.Include, } } // toVoiceAgent rebuilds an agent_yaml.VoiceAgent from the inline definition. func (d AgentDefinitionInline) toVoiceAgent() agent_yaml.VoiceAgent { return agent_yaml.VoiceAgent{ - AgentDefinition: d.AgentDefinition, - ModelType: d.ModelType, - Model: d.Model, - Instructions: d.Instructions, - Voice: d.Voice, - Store: d.Store, + AgentDefinition: d.AgentDefinition, + ModelType: d.ModelType, + Model: d.Model, + Instructions: d.Instructions, + Voice: d.Voice, + StructuredInputs: d.StructuredInputs, + Audio: d.Audio, + OutputModalities: d.OutputModalities, + Store: d.Store, + Tools: d.Tools, + Avatar: d.Avatar, + Greeting: d.Greeting, + Handoff: d.Handoff, + ToolChoice: d.ToolChoice, + ParallelToolCalls: d.ParallelToolCalls, + MaxOutputTokens: d.MaxOutputTokens, + Include: d.Include, } } diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go index 3b01eb4a811..280abab28bd 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go @@ -2223,6 +2223,13 @@ func (p *AgentServiceTargetProvider) deployVoiceAgent( fmt.Sprintf("set %s to legacy, unified, or unified-flat", voiceAgentAPIEnvKey), ) } + if hasAdvancedVoiceConfig(va) && apiMode != voiceAgentAPIModeUnifiedFlat { + return nil, exterrors.Validation( + exterrors.CodeInvalidParameter, + "advanced prompt-voice settings require the unified flat voice API mode", + fmt.Sprintf("set %s to unified-flat", voiceAgentAPIEnvKey), + ) + } var request *agent_api.CreateAgentRequest if apiMode == voiceAgentAPIModeUnifiedFlat { @@ -2299,6 +2306,20 @@ func (p *AgentServiceTargetProvider) deployVoiceAgent( return &azdext.ServiceDeployResult{Artifacts: artifacts}, nil } +func hasAdvancedVoiceConfig(va agent_yaml.VoiceAgent) bool { + return len(va.StructuredInputs) > 0 || + va.Audio != nil || + len(va.OutputModalities) > 0 || + len(va.Tools) > 0 || + len(va.Avatar) > 0 || + len(va.Greeting) > 0 || + len(va.Handoff) > 0 || + va.ToolChoice != nil || + va.ParallelToolCalls != nil || + va.MaxOutputTokens != nil || + len(va.Include) > 0 +} + func validateVoiceAgentDeployResponse(agentObject *agent_api.AgentObject, apiMode voiceAgentAPIMode) error { if agentObject == nil { return fmt.Errorf("malformed voice agent service response: missing agent object") diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go index bd4d87fa290..25120eb55fc 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go @@ -775,6 +775,14 @@ func TestShouldUpdateVoiceAgent(t *testing.T) { }) } +func TestHasAdvancedVoiceConfig(t *testing.T) { + store := false + require.False(t, hasAdvancedVoiceConfig(agent_yaml.VoiceAgent{})) + require.False(t, hasAdvancedVoiceConfig(agent_yaml.VoiceAgent{Store: &store})) + require.True(t, hasAdvancedVoiceConfig(agent_yaml.VoiceAgent{Audio: &agent_yaml.VoiceAudio{}})) + require.True(t, hasAdvancedVoiceConfig(agent_yaml.VoiceAgent{Tools: []map[string]any{{"type": "system"}}})) +} + func createSymlinkOrSkip(t *testing.T, oldname, newname string) { t.Helper() diff --git a/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json b/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json index 55a8ea2c55d..6eca9a0ef5a 100644 --- a/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json +++ b/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json @@ -64,14 +64,62 @@ "type": "string", "description": "Voice agent (kind: prompt-voice) system prompt for the assistant." }, - "voice": { - "type": "string", - "description": "Voice agent (kind: prompt-voice) output voice name (e.g. 'en-US-Ava:DragonHDLatestNeural' for an Azure Neural voice, or 'alloy' for an OpenAI realtime voice)." - }, - "store": { - "type": "boolean", - "description": "Voice agent (kind: prompt-voice) server-side logging toggle (transcript + per-turn audio). Defaults to false when omitted." - }, + "voice": { + "type": "string", + "description": "Voice agent (kind: prompt-voice) output voice name (e.g. 'en-US-Ava:DragonHDLatestNeural' for an Azure Neural voice, or 'alloy' for an OpenAI realtime voice)." + }, + "structuredInputs": { + "type": "object", + "description": "Voice agent (kind: prompt-voice) structured prompt inputs. Use description, defaultValue, schema, and required; azd maps defaultValue to the service wire field default_value.", + "additionalProperties": true + }, + "audio": { + "$ref": "#/definitions/VoiceAudio" + }, + "outputModalities": { + "type": "array", + "description": "Voice agent (kind: prompt-voice) output modalities. Well-known values are audio, text, animation, and avatar.", + "items": { "type": "string", "minLength": 1 } + }, + "store": { + "type": "boolean", + "description": "Voice agent (kind: prompt-voice) server-side logging toggle (transcript + per-turn audio). Defaults to false when omitted." + }, + "tools": { + "type": "array", + "description": "Voice agent (kind: prompt-voice) tools. Direct tool types include function, mcp, system, and toolbox.", + "items": { "type": "object", "additionalProperties": true } + }, + "avatar": { + "type": "object", + "description": "Voice agent (kind: prompt-voice) avatar configuration.", + "additionalProperties": true + }, + "greeting": { + "type": "object", + "description": "Voice agent (kind: prompt-voice) greeting configuration, such as template or llm_generated.", + "additionalProperties": true + }, + "handoff": { + "type": "object", + "description": "Voice agent (kind: prompt-voice) handoff configuration.", + "additionalProperties": true + }, + "toolChoice": { + "description": "Voice agent (kind: prompt-voice) tool choice behavior, such as none, auto, required, or a tool choice object." + }, + "parallelToolCalls": { + "type": "boolean", + "description": "Voice agent (kind: prompt-voice) parallel tool call toggle." + }, + "maxOutputTokens": { + "description": "Voice agent (kind: prompt-voice) maximum output tokens. Use an integer or a service-supported string such as inf." + }, + "include": { + "type": "array", + "description": "Voice agent (kind: prompt-voice) extra service response fields to include.", + "items": { "type": "string" } + }, "name": { "type": "string", "description": "The agent name." @@ -152,7 +200,7 @@ "required": ["protocol"], "additionalProperties": false }, - "CodeConfiguration": { + "CodeConfiguration": { "type": "object", "description": "Code deploy configuration. When present, the agent is deployed from source (ZIP) instead of a container image.", "properties": { @@ -161,9 +209,96 @@ "dependencyResolution": { "type": "string", "description": "Dependency resolution mode (e.g., 'bundled', 'remote_build')." } }, "required": ["runtime", "entryPoint"], - "additionalProperties": false - }, - "SessionConfiguration": { + "additionalProperties": false + }, + "VoiceAudio": { + "type": "object", + "description": "Prompt voice input and output audio configuration. Requires AZURE_VOICE_AGENT_API=unified-flat for deployment.", + "properties": { + "input": { "$ref": "#/definitions/VoiceAudioInput" }, + "output": { "$ref": "#/definitions/VoiceAudioOutput" } + }, + "additionalProperties": false + }, + "VoiceAudioInput": { + "type": "object", + "properties": { + "format": { "$ref": "#/definitions/VoiceAudioFormat" }, + "noiseReduction": { "$ref": "#/definitions/VoiceNoiseReduction" }, + "echoCancellation": { "type": "object", "additionalProperties": true }, + "turnDetection": { "$ref": "#/definitions/VoiceTurnDetection" }, + "transcription": { "$ref": "#/definitions/VoiceTranscription" } + }, + "additionalProperties": false + }, + "VoiceAudioOutput": { + "type": "object", + "properties": { + "format": { "$ref": "#/definitions/VoiceAudioFormat" }, + "voice": { "$ref": "#/definitions/VoiceConfig" }, + "speed": { "type": "number", "minimum": 0.25, "maximum": 1.5 } + }, + "additionalProperties": false + }, + "VoiceAudioFormat": { + "type": "object", + "properties": { + "type": { "type": "string", "enum": ["audio/pcm", "audio/pcmu", "audio/pcma"] }, + "rate": { "type": "integer", "minimum": 1 } + }, + "required": ["type"], + "additionalProperties": false + }, + "VoiceNoiseReduction": { + "type": "object", + "properties": { + "type": { "type": "string", "description": "Well-known values include near_field, far_field, and azure_deep_noise_suppression." } + }, + "required": ["type"], + "additionalProperties": false + }, + "VoiceTurnDetection": { + "type": "object", + "properties": { + "type": { "type": "string", "description": "Well-known values include server_vad, semantic_vad, and azure_semantic_vad." }, + "threshold": { "type": "number", "minimum": 0, "maximum": 1 }, + "prefixPaddingMs": { "type": "integer", "minimum": 0 }, + "silenceDurationMs": { "type": "integer", "minimum": 0 }, + "createResponse": { "type": "boolean" }, + "eagerness": { "type": "string" }, + "speechDurationMs": { "type": "integer", "minimum": 0 }, + "removeFillerWords": { "type": "boolean" }, + "interruptResponse": { "type": "boolean" }, + "languages": { "type": "array", "items": { "type": "string" } }, + "autoTruncate": { "type": "boolean" } + }, + "required": ["type"], + "additionalProperties": false + }, + "VoiceTranscription": { + "type": "object", + "properties": { + "model": { "type": "string" }, + "language": { "type": "string" }, + "prompt": { "type": "string" } + }, + "additionalProperties": false + }, + "VoiceConfig": { + "type": "object", + "properties": { + "type": { "type": "string", "description": "Voice provider type, such as openai, azure_standard, or azure-standard." }, + "name": { "type": "string" }, + "style": { "type": "string" }, + "pitch": { "type": "string" }, + "rate": { "type": "string" }, + "locale": { "type": "string" }, + "volume": { "type": "string" } + }, + "required": ["type", "name"], + "additionalProperties": false + }, + "SessionConfiguration": { "type": "object", "description": "Optional hosted-agent session runtime settings. When omitted, the service applies its defaults (idle timeout 900 seconds).", "properties": { From 5d062066fe052ca2fe0d4dbc13ea30adfd9b5d9c Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Thu, 20 Aug 2026 16:03:51 +0800 Subject: [PATCH 12/38] test(agents): avoid go fix pointer helper --- .../internal/pkg/agents/agent_yaml/map_voice_test.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go index 115538c572b..be279ddaec7 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go @@ -10,8 +10,6 @@ import ( "azureaiagent/internal/pkg/agents/agent_api" ) -func ptr[T any](v T) *T { return &v } - // --------------------------------------------------------------------------- // isOpenAIVoice / buildVoiceConfig // --------------------------------------------------------------------------- @@ -283,10 +281,13 @@ func TestCreateVoiceAgentAPIRequestFlat_AdvancedSettingsWireShape(t *testing.T) pitch := "+0Hz" rate := "+0%" volume := "+0%" + instructions := "You are {{persona}}, a concise voice assistant." + language := "en-US" + prompt := "Contoso terms" agent := VoiceAgent{ AgentDefinition: AgentDefinition{Kind: AgentKindPromptVoice, Name: "voice-advanced"}, Model: &Model{Id: "gpt-realtime"}, - Instructions: ptr("You are {{persona}}, a concise voice assistant."), + Instructions: &instructions, StructuredInputs: map[string]any{ "persona": map[string]any{"description": "Assistant persona", "defaultValue": "Ada"}, }, @@ -305,13 +306,13 @@ func TestCreateVoiceAgentAPIRequestFlat_AdvancedSettingsWireShape(t *testing.T) Languages: []string{"en-US"}, AutoTruncate: &autoTruncate, }, - Transcription: &VoiceTranscription{Model: "whisper-1", Language: ptr("en-US"), Prompt: ptr("Contoso terms")}, + Transcription: &VoiceTranscription{Model: "whisper-1", Language: &language, Prompt: &prompt}, }, Output: &VoiceAudioOutput{ Format: &VoiceAudioFormat{Type: "audio/pcm", Rate: &outRate}, Voice: &VoiceConfig{ Type: "azure_standard", Name: "en-US-AvaNeural", Style: &style, - Pitch: &pitch, Rate: &rate, Locale: ptr("en-US"), Volume: &volume, + Pitch: &pitch, Rate: &rate, Locale: &language, Volume: &volume, }, Speed: &speed, }, From 901bf2c45b00996c073e97cefe589b8c40c11033 Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:01:50 +0800 Subject: [PATCH 13/38] feat(agents): initialize hosted voice projects --- cli/azd/extensions/azure.ai.agents/README.md | 9 +- .../docs/hosted-voice-test-guide.md | 13 +- .../azure.ai.agents/internal/cmd/init.go | 57 +++++-- .../internal/cmd/init_from_code.go | 140 +++++++++++++++++- .../internal/cmd/init_from_code_test.go | 33 +++++ .../cmd/init_from_templates_helpers.go | 11 ++ .../cmd/init_from_templates_helpers_test.go | 20 +++ .../internal/cmd/init_hosted_voice.go | 39 +++++ .../internal/cmd/init_hosted_voice_test.go | 37 +++++ 9 files changed, 340 insertions(+), 19 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.agents/internal/cmd/init_hosted_voice.go create mode 100644 cli/azd/extensions/azure.ai.agents/internal/cmd/init_hosted_voice_test.go diff --git a/cli/azd/extensions/azure.ai.agents/README.md b/cli/azd/extensions/azure.ai.agents/README.md index 43977202514..c2e54b0a15f 100644 --- a/cli/azd/extensions/azure.ai.agents/README.md +++ b/cli/azd/extensions/azure.ai.agents/README.md @@ -168,8 +168,13 @@ Details: A hosted voice wrapper keeps Voice Live responsible for VAD, speech-to-text, and text-to-speech while routing conversation logic to a hosted agent in the -same Foundry project. Declare both services and reference the target by its -`azure.yaml` service name: +same Foundry project. Run interactive init from compatible Voice Bridge source +code by setting `AZD_AI_AGENT_ENABLE_PROMPT_VOICE=true` and selecting **Create a +hosted voice agent from the code in the current directory**. For CI, use +`azd ai agent init --kind hosted-voice ... --no-prompt`. + +Init generates both services and references the target by its `azure.yaml` +service name: ```yaml services: diff --git a/cli/azd/extensions/azure.ai.agents/docs/hosted-voice-test-guide.md b/cli/azd/extensions/azure.ai.agents/docs/hosted-voice-test-guide.md index 495e3ae3569..a09c8c4b977 100644 --- a/cli/azd/extensions/azure.ai.agents/docs/hosted-voice-test-guide.md +++ b/cli/azd/extensions/azure.ai.agents/docs/hosted-voice-test-guide.md @@ -42,6 +42,17 @@ same Foundry project. ## Manifest +Generate the composition interactively from compatible source code: + +```powershell +$env:AZD_AI_AGENT_ENABLE_PROMPT_VOICE = "true" +azd ai agent init +``` + +Select **Create a hosted voice agent from the code in the current directory**. +For CI, use `--kind hosted-voice` with the normal code deploy flags and +`--no-prompt`. + Use one project service, one hosted target, and one Voice wrapper. The wrapper references the target by its `azure.yaml` service name, not by a remote agent name copied into the file. @@ -306,7 +317,7 @@ the environment was restored. | Generic `invoke` | Responses/invocations | Not for WS | Not for Voice/WS; use Voice client/UI | | Delete agent | Yes | Yes | Yes for each layer; wrapper first | | `azd down` ownership cleanup | Existing behavior | Existing behavior | Follow-up for wrapper reverse cleanup | -| `azd ai agent init` scaffold | Yes | Yes | Follow-up; explicit manifest in this PR | +| `azd ai agent init` scaffold | Yes | Yes | Yes, interactive and no-prompt CI paths | ## Evidence to record diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/init.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/init.go index af9bce3a2b0..1f05d6c70fb 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/init.go @@ -787,6 +787,7 @@ func synthesizeImageManifestFile(agentName, image string, flagProtocols []string // kindFlagPromptVoice is the accepted --kind value for a declarative voice agent. const kindFlagPromptVoice = "prompt-voice" +const kindFlagHostedVoice = "hosted-voice" // synthesizeVoiceManifestFile writes a temporary declarative (managed) voice // agent manifest (kind: prompt-voice) to a temp dir and returns its path plus a @@ -1163,6 +1164,8 @@ func agentDefiningFlagsSet(flags *initFlags, srcBlocksReuse bool) bool { flags.modelDeployment != "" || flags.projectResourceId != "" || flags.image != "" || + flags.kind != "" || + flags.voice != "" || srcBlocksReuse || len(flags.protocols) > 0 } @@ -1322,11 +1325,12 @@ from code-deploy ZIP packaging (uses .gitignore syntax).`, // when the other runs first (e.g. --kind prompt-voice --image would // otherwise silently create a hosted image agent). if flags.kind != "" { - if !strings.EqualFold(flags.kind, kindFlagPromptVoice) { + if !strings.EqualFold(flags.kind, kindFlagPromptVoice) && + !strings.EqualFold(flags.kind, kindFlagHostedVoice) { return exterrors.Validation( exterrors.CodeInvalidParameter, fmt.Sprintf("unsupported --kind value %q", flags.kind), - fmt.Sprintf("the only supported --kind value is %q", kindFlagPromptVoice), + fmt.Sprintf("supported --kind values are %q and %q", kindFlagPromptVoice, kindFlagHostedVoice), ) } if !promptVoicePreviewEnabled() { @@ -1336,14 +1340,28 @@ from code-deploy ZIP packaging (uses .gitignore syntax).`, fmt.Sprintf("set %s=true to enable prompt voice init", promptVoicePreviewEnvVar), ) } - if flags.image != "" { + if strings.EqualFold(flags.kind, kindFlagHostedVoice) && flags.image != "" { + return exterrors.Validation( + exterrors.CodeInvalidParameter, + "--kind hosted-voice cannot be combined with --image", + "hosted voice init requires local Voice Bridge source code; drop --image", + ) + } + if strings.EqualFold(flags.kind, kindFlagHostedVoice) && flags.manifestPointer != "" { + return exterrors.Validation( + exterrors.CodeInvalidParameter, + "--kind hosted-voice cannot be combined with --manifest", + "hosted voice init generates the target and wrapper from local code; drop --manifest", + ) + } + if strings.EqualFold(flags.kind, kindFlagPromptVoice) && flags.image != "" { return exterrors.Validation( exterrors.CodeInvalidParameter, "--kind prompt-voice cannot be combined with --image", "a voice agent is managed and has no container image; drop --image", ) } - if flags.manifestPointer != "" { + if strings.EqualFold(flags.kind, kindFlagPromptVoice) && flags.manifestPointer != "" { return exterrors.Validation( exterrors.CodeInvalidParameter, "--kind prompt-voice cannot be combined with --manifest", @@ -1389,7 +1407,7 @@ from code-deploy ZIP packaging (uses .gitignore syntax).`, // language prompts and code scaffolding). Mirrors the --image fast path. // --kind value and --image incompatibility are validated above, before // either synthesis branch. - if flags.kind != "" && flags.manifestPointer == "" { + if strings.EqualFold(flags.kind, kindFlagPromptVoice) && flags.manifestPointer == "" { if flags.agentName == "" { return exterrors.Validation( exterrors.CodeInvalidParameter, @@ -1525,7 +1543,7 @@ from code-deploy ZIP packaging (uses .gitignore syntax).`, if findErr != nil { return findErr } - if existing != "" { + if existing != "" && !isHostedVoiceSourceDescriptor(existing) { useExisting := flags.noPrompt if !flags.noPrompt { confirmResp, promptErr := azdClient.Prompt().Confirm(ctx, &azdext.ConfirmRequest{ @@ -1679,7 +1697,13 @@ from code-deploy ZIP packaging (uses .gitignore syntax).`, } } else { // No manifest provided - prompt user for init mode - initMode, err := promptInitMode(ctx, azdClient, flags.noPrompt) + initMode := "" + var err error + if strings.EqualFold(flags.kind, kindFlagHostedVoice) { + initMode = initModeHostedVoice + } else { + initMode, err = promptInitMode(ctx, azdClient, flags.noPrompt) + } if err != nil { if exterrors.IsCancellation(err) { return exterrors.Cancelled("initialization was cancelled") @@ -1846,6 +1870,20 @@ from code-deploy ZIP packaging (uses .gitignore syntax).`, return err } + case initModeHostedVoice: + flags.kind = kindFlagHostedVoice + action := &InitFromCodeAction{ + azdClient: azdClient, + flags: flags, + httpClient: httpClient, + } + if err := action.Run(ctx); err != nil { + if exterrors.IsCancellation(err) { + return exterrors.Cancelled("initialization was cancelled") + } + return err + } + default: // initModeFromCode - use existing code in current directory action := &InitFromCodeAction{ @@ -1916,9 +1954,8 @@ from code-deploy ZIP packaging (uses .gitignore syntax).`, "Incompatible with --deploy-mode code.") cmd.Flags().StringVar(&flags.kind, "kind", "", - "Agent kind to initialize non-interactively. Currently supports 'prompt-voice' to create a "+ - "declarative (managed) voice agent, skipping template/language selection and code scaffolding. "+ - "Use --model to name the speech-to-speech model and --voice to set the output voice.") + "Agent kind to initialize. Supports 'prompt-voice' and 'hosted-voice'. "+ + "Hosted voice uses local code for a Voice Bridge target and creates a Voice wrapper.") cmd.Flags().StringVar(&flags.voice, "voice", "", "Output voice name for private prompt-voice automation. Hidden until public preview.") diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_code.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_code.go index a37a34a1beb..e6e56cbd088 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_code.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_code.go @@ -84,7 +84,8 @@ func (a *InitFromCodeAction) Run(ctx context.Context) error { // Guard against silently overwriting an existing agent definition. Reached // when the user declined the reuse prompt in RunE or bypassed it; we still // refuse in --no-prompt and confirm interactively. - if existing, statErr := findExistingAgentYaml(srcDir); statErr == nil && existing != "" { + if existing, statErr := findExistingAgentYaml(srcDir); statErr == nil && existing != "" && + !isHostedVoiceSourceDescriptor(existing) && !strings.EqualFold(a.flags.kind, kindFlagHostedVoice) { displayPath, relErr := filepath.Rel(srcDir, existing) if relErr != nil || displayPath == "" { displayPath = existing @@ -303,10 +304,25 @@ func (a *InitFromCodeAction) createDefinitionFromLocalAgent(ctx context.Context) } } - // Prompt user for supported protocols - protocols, err := promptProtocols(ctx, a.azdClient.Prompt(), a.flags.noPrompt, a.flags.protocols) - if err != nil { - return nil, err + // Hosted Voice targets implement the Voice Bridge 1.0 contract over the + // invocations_ws/1.0.0 transport. Other hosted agents retain the normal + // protocol selection and current default versions. + var protocols []agent_yaml.ProtocolVersionRecord + if strings.EqualFold(a.flags.kind, kindFlagHostedVoice) { + if len(a.flags.protocols) > 0 && + !(len(a.flags.protocols) == 1 && strings.EqualFold(a.flags.protocols[0], "invocations_ws")) { + return nil, exterrors.Validation( + exterrors.CodeInvalidParameter, + "hosted voice targets require the invocations_ws protocol", + "omit --protocol or pass --protocol invocations_ws", + ) + } + protocols = []agent_yaml.ProtocolVersionRecord{{Protocol: "invocations_ws", Version: "1.0.0"}} + } else { + protocols, err = promptProtocols(ctx, a.azdClient.Prompt(), a.flags.noPrompt, a.flags.protocols) + if err != nil { + return nil, err + } } // Step 1: Foundry project selection @@ -547,6 +563,20 @@ func (a *InitFromCodeAction) createDefinitionFromLocalAgent(ctx context.Context) Protocols: protocols, CodeConfiguration: codeConfig, } + if strings.EqualFold(a.flags.kind, kindFlagHostedVoice) { + definition.Metadata = &map[string]any{ + "voiceLiveCompatible": "true", + "bridgeProtocolVersion": "1.0", + } + definition.EnvironmentVariables = appendEnvVar(definition.EnvironmentVariables, agent_yaml.EnvironmentVariable{ + Name: "AZURE_OPENAI_ENDPOINT", + Value: "${FOUNDRY_PROJECT_ENDPOINT}/openai/v1/responses", + }) + definition.EnvironmentVariables = appendEnvVar(definition.EnvironmentVariables, agent_yaml.EnvironmentVariable{ + Name: "AZURE_OPENAI_DEPLOYMENT", + Value: "${AZURE_AI_MODEL_DEPLOYMENT_NAME}", + }) + } // An activity agent additionally advertises the friendly "activity" endpoint // guarded by BotServiceRbac. We compose this into any existing agent_endpoint @@ -792,6 +822,12 @@ func (a *InitFromCodeAction) addToProject( isCodeDeploy bool, ) error { agentName := definition.Name + agentServiceName := strings.ReplaceAll(agentName, " ", "") + if strings.EqualFold(a.flags.kind, kindFlagHostedVoice) { + if err := a.validateHostedVoiceServiceNames(ctx, agentServiceName); err != nil { + return err + } + } // If targetDir is ".", resolve the actual relative path from the project root to cwd. // This ensures azure.yaml gets the correct "project:" value when init is run from a subdirectory. if targetDir == "." { @@ -811,6 +847,10 @@ func (a *InitFromCodeAction) addToProject( Cpu: project.DefaultCpu, }, } + if strings.EqualFold(a.flags.kind, kindFlagHostedVoice) { + agentConfig.Container.Resources.Cpu = "1" + agentConfig.Container.Resources.Memory = "2Gi" + } agentConfig.Deployments = a.deploymentDetails @@ -845,7 +885,6 @@ func (a *InitFromCodeAction) addToProject( language = "csharp" } - agentServiceName := strings.ReplaceAll(agentName, " ", "") serviceConfig := &azdext.ServiceConfig{ Name: agentServiceName, RelativePath: targetDir, @@ -895,10 +934,99 @@ func (a *InitFromCodeAction) addToProject( return err } + if strings.EqualFold(a.flags.kind, kindFlagHostedVoice) { + if err := a.addHostedVoiceWrapper(ctx, agentServiceName); err != nil { + return err + } + } + printAgentAddedMessage(agentName) return nil } +func (a *InitFromCodeAction) validateHostedVoiceServiceNames(ctx context.Context, targetServiceName string) error { + response, err := a.azdClient.Project().Get(ctx, &azdext.EmptyRequest{}) + if err != nil { + return fmt.Errorf("checking existing services for hosted voice init: %w", err) + } + if response.Project == nil { + return nil + } + for _, serviceName := range []string{targetServiceName, hostedVoiceWrapperName(targetServiceName)} { + if _, exists := response.Project.Services[serviceName]; exists { + return exterrors.Validation( + exterrors.CodeInvalidParameter, + fmt.Sprintf("service %q already exists", serviceName), + "choose a different target agent name so target and wrapper service names are unique", + ) + } + } + return nil +} + +func (a *InitFromCodeAction) addHostedVoiceWrapper(ctx context.Context, targetServiceName string) error { + wrapperName := hostedVoiceWrapperName(targetServiceName) + response, err := a.azdClient.Project().Get(ctx, &azdext.EmptyRequest{}) + if err != nil || response.Project == nil { + return fmt.Errorf("loading project services before adding hosted voice wrapper: %w", err) + } + if _, exists := response.Project.Services[wrapperName]; exists { + return exterrors.Validation( + exterrors.CodeInvalidParameter, + fmt.Sprintf("hosted voice wrapper service %q already exists", wrapperName), + "choose a different target agent name so the generated wrapper service name is unique", + ) + } + projectServiceName := existingProjectServiceKey(ctx, a.azdClient) + if projectServiceName == "" { + return fmt.Errorf("cannot resolve the azure.ai.project service for hosted voice wrapper %q", wrapperName) + } + store := false + description := "Voice wrapper for hosted target " + targetServiceName + voiceAgent := agent_yaml.VoiceAgent{ + AgentDefinition: agent_yaml.AgentDefinition{ + Kind: agent_yaml.AgentKindPromptVoice, + Name: wrapperName, + Description: &description, + }, + ModelType: agent_yaml.VoiceModelTypeHostedAgent, + TargetAgent: &agent_yaml.VoiceTargetAgent{ + Service: targetServiceName, + Version: "deployed", + }, + Store: &store, + } + props, err := project.VoiceAgentDefinitionToServiceProperties(voiceAgent, nil) + if err != nil { + return err + } + if _, err := a.azdClient.Project().AddService(ctx, &azdext.AddServiceRequest{Service: &azdext.ServiceConfig{ + Name: wrapperName, + Host: AiAgentHost, + AdditionalProperties: props, + }}); err != nil { + return fmt.Errorf("adding hosted voice wrapper service: %w", err) + } + + if err := setServiceUses(ctx, a.azdClient, wrapperName, []string{projectServiceName, targetServiceName}); err != nil { + return err + } + fmt.Printf(" %s Added hosted voice wrapper %s -> %s\n", color.GreenString("+"), wrapperName, targetServiceName) + return nil +} + +func hostedVoiceWrapperName(targetServiceName string) string { + const suffix = "-voice" + base := strings.TrimRight(targetServiceName, "-") + if len(base)+len(suffix) > 63 { + base = strings.TrimRight(base[:63-len(suffix)], "-") + } + if base == "" { + base = "agent" + } + return base + suffix +} + // promptCodeConfiguration prompts the user for code deploy configuration settings. func (a *InitFromCodeAction) promptCodeConfiguration(ctx context.Context, srcDir string) (*agent_yaml.CodeConfiguration, error) { return promptCodeConfig(ctx, a.azdClient, srcDir, a.flags.noPrompt, codeDeployOptions{ diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_code_test.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_code_test.go index dd28c798dab..a62531b3220 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_code_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_code_test.go @@ -5,6 +5,7 @@ package cmd import ( "azureaiagent/internal/pkg/agents/agent_yaml" + "azureaiagent/internal/project" "context" "os" "path/filepath" @@ -126,6 +127,38 @@ func TestSanitizeAgentName(t *testing.T) { } } +func TestHostedVoiceWrapperName(t *testing.T) { + t.Parallel() + require.Equal(t, "voice-target-voice", hostedVoiceWrapperName("voice-target")) + long := strings.Repeat("a", 63) + got := hostedVoiceWrapperName(long) + require.Len(t, got, 63) + require.True(t, strings.HasSuffix(got, "-voice")) +} + +func TestAddHostedVoiceWrapper(t *testing.T) { + server := &recordingProjectServer{existing: map[string]*azdext.ServiceConfig{ + "foundry-project": {Name: "foundry-project", Host: AiProjectHost}, + }} + client := newProjectRecorderClient(t, server) + action := &InitFromCodeAction{azdClient: client} + + require.NoError(t, action.addHostedVoiceWrapper(t.Context(), "voice-target")) + + server.mu.Lock() + defer server.mu.Unlock() + require.Len(t, server.added, 1) + wrapper := server.added[0] + require.Equal(t, "voice-target-voice", wrapper.Name) + voiceAgent, found, err := project.VoiceAgentFromResolvedService(wrapper, t.TempDir()) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, agent_yaml.VoiceModelTypeHostedAgent, voiceAgent.ModelType) + require.Equal(t, "voice-target", voiceAgent.TargetAgent.Service) + require.Equal(t, "deployed", voiceAgent.TargetAgent.Version) + require.ElementsMatch(t, []string{"foundry-project", "voice-target"}, server.uses[wrapper.Name]) +} + func TestNormalizeForFuzzyMatch(t *testing.T) { t.Parallel() diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_templates_helpers.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_templates_helpers.go index 305ce1ba9b5..d5e8429ab8b 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_templates_helpers.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_templates_helpers.go @@ -107,6 +107,9 @@ const ( // (managed) voice agent. It maps to the same synthesized-manifest fast path // as `azd ai agent init --kind prompt-voice`. initModeVoice = "prompt_voice" + // initModeHostedVoice creates a Voice Bridge hosted target from local code + // plus a declarative Voice wrapper that references the target service. + initModeHostedVoice = "hosted_voice" ) // voiceInitChoice is the interactive menu entry for creating a prompt voice agent. @@ -117,6 +120,11 @@ var voiceInitChoice = &azdext.SelectChoice{ Value: initModeVoice, } +var hostedVoiceInitChoice = &azdext.SelectChoice{ + Label: "Create a hosted voice agent from the code in the current directory", + Value: initModeHostedVoice, +} + // promptInitMode asks the user whether to use existing code, start from a // template, or create a prompt voice agent. // If the current directory is empty, the "use existing code" option is omitted @@ -157,6 +165,9 @@ func promptInitMode(ctx context.Context, azdClient *azdext.AzdClient, noPrompt b } if voicePreviewEnabled { choices = append(choices, voiceInitChoice) + if !empty { + choices = append(choices, hostedVoiceInitChoice) + } } defaultIndex := int32(0) diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_templates_helpers_test.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_templates_helpers_test.go index 117ec3e8f35..964f581e3db 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_templates_helpers_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_templates_helpers_test.go @@ -282,6 +282,26 @@ func TestPromptInitMode_ShowsVoiceChoiceWhenPreviewEnabled(t *testing.T) { require.Equal(t, "Create a prompt voice agent", prompts.lastSelect.Options.Choices[1].Label) } +func TestPromptInitMode_ShowsHostedVoiceChoiceForLocalCode(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + t.Setenv(promptVoicePreviewEnvVar, "true") + require.NoError(t, os.WriteFile(filepath.Join(dir, "Program.cs"), []byte("class Program {}\n"), 0600)) + + prompts := &helpersPromptServer{selectIndex: 3} + azdClient := newHelpersTestAzdClient(t, &helpersProjectServer{}, prompts) + + mode, err := promptInitMode(t.Context(), azdClient, false) + + require.NoError(t, err) + require.Equal(t, initModeHostedVoice, mode) + require.NotNil(t, prompts.lastSelect) + require.Len(t, prompts.lastSelect.Options.Choices, 4) + require.Equal(t, + "Create a hosted voice agent from the code in the current directory", + prompts.lastSelect.Options.Choices[3].Label) +} + func TestFindRecommendedIndex(t *testing.T) { t.Parallel() diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_hosted_voice.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_hosted_voice.go new file mode 100644 index 00000000000..a6603d8dddf --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_hosted_voice.go @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "os" + "slices" + "strings" + + "gopkg.in/yaml.v3" +) + +// hostedVoiceSourceDescriptor is source capability metadata used by the Voice +// Bridge samples. It is not an azd agent definition and must not be adopted or +// overwritten by init. +type hostedVoiceSourceDescriptor struct { + Protocols []string `yaml:"protocols"` + VoiceLiveCompatible string `yaml:"voiceLiveCompatible"` + BridgeProtocolVersion string `yaml:"bridgeProtocolVersion"` +} + +func isHostedVoiceSourceDescriptor(path string) bool { + content, err := os.ReadFile(path) //nolint:gosec // path was discovered under the selected source directory + if err != nil { + return false + } + var descriptor hostedVoiceSourceDescriptor + if yaml.Unmarshal(content, &descriptor) != nil { + return false + } + protocols := make([]string, 0, len(descriptor.Protocols)) + for _, protocol := range descriptor.Protocols { + protocols = append(protocols, strings.ToLower(strings.TrimSpace(protocol))) + } + return slices.Contains(protocols, "invocations_ws") && + strings.EqualFold(strings.TrimSpace(descriptor.VoiceLiveCompatible), "true") && + strings.TrimSpace(descriptor.BridgeProtocolVersion) == "1.0" +} diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_hosted_voice_test.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_hosted_voice_test.go new file mode 100644 index 00000000000..44cfc4e7ff2 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_hosted_voice_test.go @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIsHostedVoiceSourceDescriptor(t *testing.T) { + t.Parallel() + path := filepath.Join(t.TempDir(), "agent.manifest.yaml") + require.NoError(t, os.WriteFile(path, []byte(` +name: voice-hosted-agent-dotnet +protocols: + - invocations_ws +voiceLiveCompatible: "true" +bridgeProtocolVersion: "1.0" +`), 0600)) + require.True(t, isHostedVoiceSourceDescriptor(path)) +} + +func TestIsHostedVoiceSourceDescriptorRejectsIncompatibleDescriptor(t *testing.T) { + t.Parallel() + path := filepath.Join(t.TempDir(), "agent.manifest.yaml") + require.NoError(t, os.WriteFile(path, []byte(` +protocols: + - responses +voiceLiveCompatible: "true" +bridgeProtocolVersion: "2.0" +`), 0600)) + require.False(t, isHostedVoiceSourceDescriptor(path)) +} From d36bc54e6c94d9db170ec8b51645e67fe8d9eafb Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:07:28 +0800 Subject: [PATCH 14/38] docs(agents): hide advanced voice private docs --- cli/azd/extensions/azure.ai.agents/README.md | 92 -------------------- 1 file changed, 92 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/README.md b/cli/azd/extensions/azure.ai.agents/README.md index 8dd303f82a6..4ddac596f9a 100644 --- a/cli/azd/extensions/azure.ai.agents/README.md +++ b/cli/azd/extensions/azure.ai.agents/README.md @@ -199,98 +199,6 @@ keys throughout this block (`invocations_moderation`, `response_mode`, `input_paths`, `stream_selectors`, `event_type`, and so on). The **values** (`non_streaming`, `streaming`, `both`, `json`, `text`) are the same in both. -## Prompt voice advanced configuration - -Advanced prompt voice settings are authored on the `azure.ai.agent` service in -`azure.yaml` and require the unified flat API mode: - -```bash -azd env set AZURE_VOICE_AGENT_API unified-flat -``` - -```yaml -services: - voice-agent: - host: azure.ai.agent - kind: prompt-voice - name: voice-agent - modelType: managed # or self_deployed for BYOM - model: - id: gpt-realtime - instructions: You are {{persona}}, a concise voice assistant. - structuredInputs: - persona: - description: Assistant persona - defaultValue: Ada - schema: - type: string - audio: - input: - format: - type: audio/pcmu - noiseReduction: - type: near_field - echoCancellation: - type: server_echo_cancellation - reference_source: server - channels: 1 - turnDetection: - type: azure_semantic_vad - threshold: 0.6 - speechDurationMs: 120 - removeFillerWords: true - createResponse: true - interruptResponse: true - languages: [en-US] - autoTruncate: true - transcription: - model: whisper-1 - language: en-US - output: - format: - type: audio/pcm - rate: 24000 - voice: - type: azure_standard - name: en-US-AvaNeural - locale: en-US - style: cheerful - speed: 1.0 - outputModalities: [audio, text] - tools: - - type: system - name: end_conversation - avatar: - type: video_avatar - character: lisa - style: casual-sitting - output_protocol: webrtc - greeting: - type: template - text: Hello {{persona}} - toolChoice: auto - parallelToolCalls: true - maxOutputTokens: inf - include: - - item.input_audio_transcription.phrases -``` - -Notes: - -- `voice`, `instructions`, and `store` remain supported for simple prompt voice - agents. Missing audio fields keep the existing azd defaults. -- `audio.output.voice` uses an author-friendly object shape; in `unified-flat` - mode azd maps it to the service flat fields `voice`, `voice_type`, - `voice_locale`, `style`, `pitch`, `rate`, and `volume`. -- `structuredInputs.defaultValue` maps to the service wire field - `default_value`. -- Direct voice tool types are `function`, `mcp`, `system`, and `toolbox`. - Server-side tools such as `web_search`, `azure_ai_search`, and `openapi` must - be packaged through a toolbox. -- `tools`, `avatar`, `greeting`, `handoff`, `toolChoice`, and - `echoCancellation` intentionally remain light pass-through blocks so azd does - not block new service-side additions. - ## Session idle timeout A hosted agent's runtime session sandbox is suspended by Foundry after a period From ae5f687037f9bd818f36a907b0e2c4aac9f75cb7 Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Fri, 21 Aug 2026 17:40:54 +0800 Subject: [PATCH 15/38] feat(agents): initialize hosted voice samples --- .../azure.ai.agents/internal/cmd/init.go | 42 ++++++++++ .../internal/cmd/init_hosted_voice.go | 82 +++++++++++++++++-- .../internal/cmd/init_hosted_voice_test.go | 65 +++++++++++++++ 3 files changed, 184 insertions(+), 5 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/init.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/init.go index 1f05d6c70fb..b9bb7b28c1c 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/init.go @@ -371,6 +371,16 @@ func resolveAgentNameFromManifestPointer( flags.agentName = validated return validated, nil } + if descriptor, compatible, err := loadHostedVoiceSourceDescriptor(manifestPointer); err != nil { + return "", fmt.Errorf("reading hosted voice sample descriptor: %w", err) + } else if compatible { + validated, err := validateInitAgentName(descriptor.Name) + if err != nil { + return "", err + } + flags.agentName = validated + return validated, nil + } peeked := peekManifestName(ctx, manifestPointer, httpClient) if peeked == "" { @@ -1071,6 +1081,38 @@ func runInitFromManifest( createdFolderDisplay string, userProvidedManifest bool, ) error { + if descriptor, compatible, err := loadHostedVoiceSourceDescriptor(flags.manifestPointer); err != nil { + return fmt.Errorf("reading hosted voice sample descriptor: %w", err) + } else if compatible { + cwd, err := os.Getwd() + if err != nil { + return fmt.Errorf("getting current directory for hosted voice sample: %w", err) + } + if !isSamePath(filepath.Dir(flags.manifestPointer), cwd) { + return exterrors.Validation( + exterrors.CodeInvalidParameter, + "hosted voice sample descriptor must be initialized from its source directory", + fmt.Sprintf("change directory to %q and run 'azd ai agent init -m %s'", filepath.Dir(flags.manifestPointer), filepath.Base(flags.manifestPointer)), + ) + } + if !promptVoicePreviewEnabled() { + return exterrors.Validation( + exterrors.CodeInvalidParameter, + "hosted voice agent init is private preview", + fmt.Sprintf("set %s=true to enable hosted voice init", promptVoicePreviewEnvVar), + ) + } + if err := applyHostedVoiceSourceDescriptor(flags, flags.manifestPointer, descriptor); err != nil { + return err + } + action := &InitFromCodeAction{ + azdClient: azdClient, + flags: flags, + httpClient: httpClient, + } + return action.Run(ctx) + } + // Ensure project and environment exist (no subscription/location prompting yet) projectConfig, err := ensureProject(ctx, flags, azdClient, targetDir) if err != nil { diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_hosted_voice.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_hosted_voice.go index a6603d8dddf..8999434d1ae 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_hosted_voice.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_hosted_voice.go @@ -4,7 +4,11 @@ package cmd import ( + "azureaiagent/internal/exterrors" + + "fmt" "os" + "path/filepath" "slices" "strings" @@ -15,25 +19,93 @@ import ( // Bridge samples. It is not an azd agent definition and must not be adopted or // overwritten by init. type hostedVoiceSourceDescriptor struct { + Name string `yaml:"name"` Protocols []string `yaml:"protocols"` VoiceLiveCompatible string `yaml:"voiceLiveCompatible"` BridgeProtocolVersion string `yaml:"bridgeProtocolVersion"` } -func isHostedVoiceSourceDescriptor(path string) bool { +func loadHostedVoiceSourceDescriptor(path string) (*hostedVoiceSourceDescriptor, bool, error) { + if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") { + return nil, false, nil + } content, err := os.ReadFile(path) //nolint:gosec // path was discovered under the selected source directory if err != nil { - return false + if os.IsNotExist(err) { + return nil, false, nil + } + return nil, false, err + } + var raw map[string]any + if err := yaml.Unmarshal(content, &raw); err != nil { + return nil, false, nil + } + if _, isAzdManifest := raw["template"]; isAzdManifest { + return nil, false, nil } var descriptor hostedVoiceSourceDescriptor - if yaml.Unmarshal(content, &descriptor) != nil { - return false + if err := yaml.Unmarshal(content, &descriptor); err != nil { + return nil, false, nil } protocols := make([]string, 0, len(descriptor.Protocols)) for _, protocol := range descriptor.Protocols { protocols = append(protocols, strings.ToLower(strings.TrimSpace(protocol))) } - return slices.Contains(protocols, "invocations_ws") && + compatible := strings.TrimSpace(descriptor.Name) != "" && + slices.Contains(protocols, "invocations_ws") && strings.EqualFold(strings.TrimSpace(descriptor.VoiceLiveCompatible), "true") && strings.TrimSpace(descriptor.BridgeProtocolVersion) == "1.0" + return &descriptor, compatible, nil +} + +func isHostedVoiceSourceDescriptor(path string) bool { + _, compatible, err := loadHostedVoiceSourceDescriptor(path) + return err == nil && compatible +} + +func applyHostedVoiceSourceDescriptor(flags *initFlags, path string, descriptor *hostedVoiceSourceDescriptor) error { + if flags == nil || descriptor == nil { + return fmt.Errorf("hosted voice source descriptor is required") + } + name, err := validateInitAgentName(descriptor.Name) + if err != nil { + return err + } + sourceDir, err := filepath.Abs(filepath.Dir(path)) + if err != nil { + return fmt.Errorf("resolving hosted voice sample directory: %w", err) + } + flags.kind = kindFlagHostedVoice + if strings.TrimSpace(flags.agentName) == "" { + flags.agentName = name + } + flags.src = sourceDir + flags.deployMode = "code" + flags.protocols = []string{"invocations_ws"} + flags.depResolution = "bundled" + if isDotnetProject(sourceDir) { + flags.runtime = "dotnet_10" + } else if isPythonProject(sourceDir) { + flags.runtime = "python_3_13" + } + if flags.runtime == "" { + return exterrors.Validation( + exterrors.CodeInvalidParameter, + "could not detect a supported runtime for the hosted voice sample", + "place agent.manifest.yaml next to a .NET or Python agent project", + ) + } + flags.entryPoint = detectDefaultEntryPoint(sourceDir, flags.runtime) + validEntryPoint := strings.TrimSpace(flags.entryPoint) != "" + if flags.runtime != "dotnet_10" { + validEntryPoint = validEntryPoint && fileExists(filepath.Join(sourceDir, flags.entryPoint)) + } + if !validEntryPoint { + return exterrors.Validation( + exterrors.CodeInvalidParameter, + "could not detect the hosted voice sample entry point", + "ensure the sample contains its expected .NET assembly or Python entry module", + ) + } + return nil } diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_hosted_voice_test.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_hosted_voice_test.go index 44cfc4e7ff2..9aa4989246c 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_hosted_voice_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_hosted_voice_test.go @@ -35,3 +35,68 @@ bridgeProtocolVersion: "2.0" `), 0600)) require.False(t, isHostedVoiceSourceDescriptor(path)) } + +func TestApplyHostedVoiceSourceDescriptorDotnet(t *testing.T) { + t.Parallel() + dir := t.TempDir() + manifestPath := filepath.Join(dir, "agent.manifest.yaml") + require.NoError(t, os.WriteFile(manifestPath, []byte("name: voice-sample\n"), 0600)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "VoiceHostedAgent.csproj"), []byte(` +VoiceHostedAgent +`), 0600)) + flags := &initFlags{} + err := applyHostedVoiceSourceDescriptor(flags, manifestPath, &hostedVoiceSourceDescriptor{Name: "voice-sample"}) + require.NoError(t, err) + require.Equal(t, kindFlagHostedVoice, flags.kind) + require.Equal(t, "voice-sample", flags.agentName) + require.Equal(t, dir, flags.src) + require.Equal(t, "code", flags.deployMode) + require.Equal(t, "dotnet_10", flags.runtime) + require.Equal(t, "VoiceHostedAgent.dll", flags.entryPoint) + require.Equal(t, "bundled", flags.depResolution) + require.Equal(t, []string{"invocations_ws"}, flags.protocols) +} + +func TestApplyHostedVoiceSourceDescriptorPreservesExplicitName(t *testing.T) { + t.Parallel() + dir := t.TempDir() + manifestPath := filepath.Join(dir, "agent.manifest.yaml") + require.NoError(t, os.WriteFile(manifestPath, []byte("name: descriptor-name\n"), 0600)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "VoiceHostedAgent.csproj"), []byte(` +VoiceHostedAgent +`), 0600)) + flags := &initFlags{agentName: "explicit-name"} + require.NoError(t, applyHostedVoiceSourceDescriptor( + flags, manifestPath, &hostedVoiceSourceDescriptor{Name: "descriptor-name"}, + )) + require.Equal(t, "explicit-name", flags.agentName) +} + +func TestApplyHostedVoiceSourceDescriptorRejectsMissingEntrypoint(t *testing.T) { + t.Parallel() + dir := t.TempDir() + manifestPath := filepath.Join(dir, "agent.manifest.yaml") + require.NoError(t, os.WriteFile(manifestPath, []byte("name: voice-sample\n"), 0600)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "requirements.txt"), []byte("websockets\n"), 0600)) + err := applyHostedVoiceSourceDescriptor( + &initFlags{}, manifestPath, &hostedVoiceSourceDescriptor{Name: "voice-sample"}, + ) + require.ErrorContains(t, err, "could not detect the hosted voice sample entry point") +} + +func TestHostedVoiceDescriptorDoesNotMatchAzdManifest(t *testing.T) { + t.Parallel() + path := filepath.Join(t.TempDir(), "agent.yaml") + require.NoError(t, os.WriteFile(path, []byte(` +name: manifest +protocols: [invocations_ws] +voiceLiveCompatible: "true" +bridgeProtocolVersion: "1.0" +template: + kind: hosted + name: agent +`), 0600)) + _, compatible, err := loadHostedVoiceSourceDescriptor(path) + require.NoError(t, err) + require.False(t, compatible) +} From 9f5bcb8c798fd3572fae9cf18382b63b60985c62 Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Fri, 21 Aug 2026 18:57:19 +0800 Subject: [PATCH 16/38] fix(agents): validate voice transcription includes --- .../internal/pkg/agents/agent_yaml/parse.go | 24 +++++++++++++++++-- .../pkg/agents/agent_yaml/parse_voice_test.go | 21 ++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go index 42e894667fc..a90fbe6046d 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go @@ -489,8 +489,9 @@ func validateVoiceAgentAdvancedConfig(agent VoiceAgent) []string { } if agent.Audio == nil { - return errors + return append(errors, validateVoiceIncludeTranscriptionCompatibility(agent, "")...) } + transcriptionModel := "" if agent.Audio.Input != nil { errors = append(errors, validateVoiceAudioFormat("template.audio.input.format", agent.Audio.Input.Format)...) if nr := agent.Audio.Input.NoiseReduction; nr != nil && strings.TrimSpace(nr.Type) == "" { @@ -513,6 +514,9 @@ func validateVoiceAgentAdvancedConfig(agent VoiceAgent) []string { errors = append(errors, "template.audio.input.turn_detection.speech_duration_ms must be >= 0") } } + if agent.Audio.Input.Transcription != nil { + transcriptionModel = agent.Audio.Input.Transcription.Model + } } if agent.Audio.Output != nil { errors = append(errors, validateVoiceAudioFormat("template.audio.output.format", agent.Audio.Output.Format)...) @@ -528,7 +532,23 @@ func validateVoiceAgentAdvancedConfig(agent VoiceAgent) []string { errors = append(errors, "template.audio.output.speed must be between 0.25 and 1.5") } } - return errors + return append(errors, validateVoiceIncludeTranscriptionCompatibility(agent, transcriptionModel)...) +} + +func validateVoiceIncludeTranscriptionCompatibility(agent VoiceAgent, transcriptionModel string) []string { + if !slices.Contains(agent.Include, "item.input_audio_transcription.phrases") { + return nil + } + model := strings.TrimSpace(transcriptionModel) + if model == "" { + model = defaultVoiceInputTranscriptionModel + } + if model == "azure-speech" || model == "azure-fast-transcription" { + return nil + } + return []string{ + "template.include item.input_audio_transcription.phrases requires template.audio.input.transcription.model to be azure-speech or azure-fast-transcription", + } } func validateVoiceAudioFormat(path string, format *VoiceAudioFormat) []string { diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse_voice_test.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse_voice_test.go index 6aba8a368dd..b348d867651 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse_voice_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse_voice_test.go @@ -118,3 +118,24 @@ model_type: unsupported t.Fatalf("expected invalid model_type error, got: %v", err) } } +func TestValidateAgentDefinition_PromptVoice_InvalidIncludeTranscriptionModel(t *testing.T) { + yamlContent := []byte(` +kind: prompt-voice +name: voice-agent +model: + id: gpt-realtime +audio: + input: + transcription: + model: whisper-1 +include: + - item.input_audio_transcription.phrases +`) + err := ValidateAgentDefinition(yamlContent) + if err == nil { + t.Fatal("expected include/transcription validation error") + } + if !strings.Contains(err.Error(), "azure-speech") || !strings.Contains(err.Error(), "azure-fast-transcription") { + t.Fatalf("expected transcription model guidance in error, got: %v", err) + } +} From 348edb407ea5d85cedd1714c11b72d04db52a626 Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Mon, 24 Aug 2026 16:35:29 +0800 Subject: [PATCH 17/38] refactor(agents): use unified voice API by default --- .../internal/pkg/agents/agent_api/models.go | 13 ++- .../pkg/agents/agent_api/operations.go | 29 ----- .../pkg/agents/agent_api/operations_test.go | 64 ----------- .../internal/pkg/agents/agent_yaml/map.go | 49 +++------ .../pkg/agents/agent_yaml/map_voice_test.go | 20 ++-- .../internal/project/service_target_agent.go | 102 +++--------------- .../project/service_target_agent_test.go | 59 +--------- 7 files changed, 50 insertions(+), 286 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go index 9fedb6e56e3..946ac70951d 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go @@ -69,10 +69,9 @@ const ( AgentKindHosted AgentKind = "hosted" AgentKindWorkflow AgentKind = "workflow" // AgentKindVoice is the data-plane (service) kind for a declarative voice - // (speech-to-speech) agent. Note: this is the wire value posted to the - // /voice_agents collection. The azd manifest authoring kind is - // "prompt-voice" (agent_yaml.AgentKindPromptVoice), which the map layer - // translates to this value. + // (speech-to-speech) agent. The azd manifest authoring kind is "prompt-voice" + // (agent_yaml.AgentKindPromptVoice), which the map layer translates to this + // value before posting through the unified /agents API. AgentKindVoice AgentKind = "voice" ) @@ -420,9 +419,9 @@ type VoiceAudioConfigFlat struct { Output *VoiceOutputConfigFlat `json:"output,omitempty"` } -// VoiceAgentDefinition is the data-plane definition body POSTed to the -// /voice_agents collection for a declarative (managed) voice agent. Its Kind -// is always AgentKindVoice ("voice"). +// VoiceAgentDefinition is retained for compatibility with object-shaped voice +// definitions returned by older services. New prompt voice deploys use +// VoiceAgentDefinitionFlat. type VoiceAgentDefinition struct { AgentDefinition ModelType VoiceModelType `json:"model_type"` diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations.go index fbd6f514ef3..ce9aff37322 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations.go @@ -216,35 +216,6 @@ func (c *AgentClient) doVoiceJSONAgentRequest( return &agent, nil } -// CreateVoiceAgent creates a new declarative (managed) voice agent. -// -// Voice agents live in a separate data-plane collection (/voice_agents), distinct -// from the /agents collection used by hosted/workflow agents. The request -// Definition must be a *VoiceAgentDefinition (service kind "voice"). -// -// overriddenHost, when non-empty, is sent as the x-ms-overridden-host header. -// This routes the request directly to the regional Hyena data-plane host, -// bypassing the public Foundry APIM (whose voice route may not yet be rolled -// out). Pass "" to use the default endpoint routing. -// -// Redeploy semantics: the voice data-plane exposes create-only POST /voice_agents -// with no version/upsert model (unlike hosted agents, which mint a new -// agent-version per deploy). A second `azd deploy` of the same voice service -// therefore re-POSTs with the same name and the service rejects it with a -// non-success status, which this method surfaces as a deploy error rather than -// silently overwriting the existing agent. Idempotent redeploy/update is tracked -// as a follow-up (see the PR "Follow-ups" section); until the service adds an -// update route, redeploy requires deleting the existing voice agent first. -func (c *AgentClient) CreateVoiceAgent( - ctx context.Context, - request *CreateAgentRequest, - apiVersion string, - overriddenHost string, -) (*AgentObject, error) { - url := fmt.Sprintf("%s/voice_agents?api-version=%s", c.endpoint, apiVersion) - return c.doVoiceJSONAgentRequest(ctx, http.MethodPost, url, request, overriddenHost) -} - // GetVoiceAgentUnified retrieves a voice agent through the unified /agents // endpoint with the voice preview opt-in header. Use this instead of GetAgent // when deciding whether to create or update a prompt voice agent. diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations_test.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations_test.go index 7aacd0d7ba5..d47e3a5dee1 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations_test.go @@ -919,35 +919,6 @@ func TestDownloadAgentCode_ReturnsErrorOnNon200(t *testing.T) { require.Error(t, err) } -func TestCreateVoiceAgent_PostsToVoiceCollectionWithPreviewHeader(t *testing.T) { - body := `{"object":"agent","id":"va-1","name":"my-voice","versions":{"latest":{}}}` - client, transport := newCaptureClient(http.StatusOK, body) - - agent, err := client.CreateVoiceAgent( - t.Context(), - &CreateAgentRequest{Name: "my-voice"}, - AgentEndpointAPIVersion, - "", - ) - - require.NoError(t, err) - require.Equal(t, "my-voice", agent.Name) - - require.Len(t, transport.requests, 1) - req := transport.requests[0] - - require.Equal(t, http.MethodPost, req.Method) - require.Equal(t, "/api/projects/proj/voice_agents", req.URL.Path) - require.Equal(t, AgentEndpointAPIVersion, req.URL.Query().Get("api-version")) - require.Equal(t, voiceAgentsPreviewFeature, req.Header.Get("Foundry-Features")) - // Default routing: no host override. - require.Empty(t, req.Header.Get("x-ms-overridden-host")) - - reqBody, err := io.ReadAll(req.Body) - require.NoError(t, err) - require.Contains(t, string(reqBody), `"name":"my-voice"`) -} - func TestCreateVoiceAgentUnified_PostsToAgentsWithPreviewHeader(t *testing.T) { body := `{"object":"agent","id":"va-1","name":"my-voice","versions":{"latest":{}}}` client, transport := newCaptureClient(http.StatusOK, body) @@ -1014,38 +985,3 @@ func TestUpdateVoiceAgentUnified_PostsToNamedAgentWithPreviewHeader(t *testing.T require.Equal(t, voiceAgentsPreviewFeature, req.Header.Get("Foundry-Features")) require.Equal(t, "regional.hyena.example.com", req.Header.Get("x-ms-overridden-host")) } - -func TestCreateVoiceAgent_SetsOverriddenHostHeader(t *testing.T) { - client, transport := newCaptureClient(http.StatusCreated, `{"name":"my-voice","versions":{"latest":{}}}`) - - _, err := client.CreateVoiceAgent( - t.Context(), - &CreateAgentRequest{Name: "my-voice"}, - AgentEndpointAPIVersion, - "regional.hyena.example.com", - ) - - require.NoError(t, err) - require.Len(t, transport.requests, 1) - require.Equal( - t, - "regional.hyena.example.com", - transport.requests[0].Header.Get("x-ms-overridden-host"), - ) -} - -func TestCreateVoiceAgent_ReturnsErrorOnNonSuccess(t *testing.T) { - client, _ := newCaptureClient( - http.StatusForbidden, - `{"error":{"code":"preview_feature_required","message":"voice agents preview"}}`, - ) - - _, err := client.CreateVoiceAgent( - t.Context(), - &CreateAgentRequest{Name: "my-voice"}, - AgentEndpointAPIVersion, - "", - ) - - require.Error(t, err) -} diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go index 42b1463fb05..b2c9ae5cb9d 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go @@ -608,16 +608,18 @@ func flatVoiceLocale(voice *agent_api.VoiceConfig) string { // voice agent. It translates the authoring kind "prompt-voice" into the // data-plane service kind "voice" and defaults the audio pipeline. func CreateVoiceAgentAPIRequest(voiceAgent VoiceAgent) (*agent_api.CreateAgentRequest, error) { - return createVoiceAgentAPIRequest(voiceAgent, false) + return createVoiceAgentAPIRequest(voiceAgent) } -// CreateVoiceAgentAPIRequestFlat builds a CreateAgentRequest using the newer -// TiP/unified API flat output voice shape. +// CreateVoiceAgentAPIRequestFlat is kept as a compatibility wrapper for tests +// and callers that explicitly selected the transition name while the unified API +// work was in flight. Prompt voice requests now always use the unified flat +// output shape. func CreateVoiceAgentAPIRequestFlat(voiceAgent VoiceAgent) (*agent_api.CreateAgentRequest, error) { - return createVoiceAgentAPIRequest(voiceAgent, true) + return createVoiceAgentAPIRequest(voiceAgent) } -func createVoiceAgentAPIRequest(voiceAgent VoiceAgent, flatOutput bool) (*agent_api.CreateAgentRequest, error) { +func createVoiceAgentAPIRequest(voiceAgent VoiceAgent) (*agent_api.CreateAgentRequest, error) { modelID := "" if voiceAgent.Model != nil { modelID = strings.TrimSpace(voiceAgent.Model.Id) @@ -657,32 +659,7 @@ func createVoiceAgentAPIRequest(voiceAgent VoiceAgent, flatOutput bool) (*agent_ Transcription: &agent_api.VoiceTranscription{Model: defaultVoiceInputTranscriptionModel}, } voiceConfig := buildVoiceConfig(voiceName) - if flatOutput { - voiceDef := agent_api.VoiceAgentDefinitionFlat{ - AgentDefinition: agent_api.AgentDefinition{ - // Translate authoring kind prompt-voice -> service kind voice. - Kind: agent_api.AgentKindVoice, - }, - ModelType: modelType, - Model: modelID, - Instructions: instructions, - Audio: &agent_api.VoiceAudioConfigFlat{ - Input: input, - Output: &agent_api.VoiceOutputConfigFlat{ - Format: audioFormat, - Voice: voiceConfig.Name, - VoiceType: flatVoiceType(voiceConfig), - VoiceLocale: flatVoiceLocale(voiceConfig), - }, - }, - OutputModalities: []string{"audio"}, - Store: voiceAgent.Store, - } - - return createAgentAPIRequest(voiceAgent.AgentDefinition, voiceDef, nil, nil) - } - - voiceDef := agent_api.VoiceAgentDefinition{ + voiceDef := agent_api.VoiceAgentDefinitionFlat{ AgentDefinition: agent_api.AgentDefinition{ // Translate authoring kind prompt-voice -> service kind voice. Kind: agent_api.AgentKindVoice, @@ -690,11 +667,13 @@ func createVoiceAgentAPIRequest(voiceAgent VoiceAgent, flatOutput bool) (*agent_ ModelType: modelType, Model: modelID, Instructions: instructions, - Audio: &agent_api.VoiceAudioConfig{ + Audio: &agent_api.VoiceAudioConfigFlat{ Input: input, - Output: &agent_api.VoiceOutputConfig{ - Format: audioFormat, - Voice: voiceConfig, + Output: &agent_api.VoiceOutputConfigFlat{ + Format: audioFormat, + Voice: voiceConfig.Name, + VoiceType: flatVoiceType(voiceConfig), + VoiceLocale: flatVoiceLocale(voiceConfig), }, }, OutputModalities: []string{"audio"}, diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go index 1e536b344da..4b6e78ae065 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go @@ -90,9 +90,9 @@ func TestCreateVoiceAgentAPIRequest_Defaults(t *testing.T) { t.Errorf("Name = %q", req.Name) } - def, ok := req.Definition.(agent_api.VoiceAgentDefinition) + def, ok := req.Definition.(agent_api.VoiceAgentDefinitionFlat) if !ok { - t.Fatalf("expected VoiceAgentDefinition, got %T", req.Definition) + t.Fatalf("expected VoiceAgentDefinitionFlat, got %T", req.Definition) } // Authoring kind prompt-voice is translated to service kind voice. @@ -131,9 +131,9 @@ func TestCreateVoiceAgentAPIRequest_Defaults(t *testing.T) { if out.Format == nil || out.Format.Type != defaultVoiceAudioType || out.Format.Rate != defaultVoiceAudioRate { t.Errorf("output format = %+v", out.Format) } - // Default voice is the DragonHD Azure Neural voice. - if out.Voice == nil || out.Voice.Type != "azure_standard" || out.Voice.Name != defaultVoiceName { - t.Errorf("output voice = %+v, want azure_standard/%s", out.Voice, defaultVoiceName) + // Default voice is the DragonHD Azure Neural voice in the flat unified shape. + if out.Voice != defaultVoiceName || out.VoiceType != "azure-standard" || out.VoiceLocale != "en-US" { + t.Errorf("output voice = %+v, want azure-standard/%s", out, defaultVoiceName) } // Store defaults to nil (service defaults to false). if def.Store != nil { @@ -160,12 +160,12 @@ func TestCreateVoiceAgentAPIRequest_Overrides(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - def := req.Definition.(agent_api.VoiceAgentDefinition) + def := req.Definition.(agent_api.VoiceAgentDefinitionFlat) if def.Instructions != instructions { t.Errorf("Instructions = %q", def.Instructions) } // "alloy" is an OpenAI realtime voice. - if def.Audio.Output.Voice.Type != "openai" || def.Audio.Output.Voice.Name != "alloy" { + if def.Audio.Output.VoiceType != "openai" || def.Audio.Output.Voice != "alloy" { t.Errorf("voice = %+v, want openai/alloy", def.Audio.Output.Voice) } if def.Store == nil || !*def.Store { @@ -276,7 +276,7 @@ func TestCreateVoiceAgentAPIRequest_ExplicitManaged(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - if req.Definition.(agent_api.VoiceAgentDefinition).ModelType != agent_api.VoiceModelTypeManaged { + if req.Definition.(agent_api.VoiceAgentDefinitionFlat).ModelType != agent_api.VoiceModelTypeManaged { t.Errorf("ModelType not managed") } } @@ -304,7 +304,7 @@ func TestCreateVoiceAgentAPIRequest_TrimsModelID(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - def := req.Definition.(agent_api.VoiceAgentDefinition) + def := req.Definition.(agent_api.VoiceAgentDefinitionFlat) if def.Model != "my-realtime-deployment" { t.Errorf("Model = %q, want trimmed model id", def.Model) } @@ -334,7 +334,7 @@ func TestCreateVoiceAgentAPIRequest_SelfDeployedMapped(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - def := req.Definition.(agent_api.VoiceAgentDefinition) + def := req.Definition.(agent_api.VoiceAgentDefinitionFlat) if def.ModelType != agent_api.VoiceModelTypeSelfDeployed { t.Errorf("ModelType = %q, want self_deployed", def.ModelType) } diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go index 3b01eb4a811..51f590ce0cf 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go @@ -2155,57 +2155,17 @@ func (p *AgentServiceTargetProvider) deployHostedAgent( }, nil } -// voiceOverriddenHostEnvKey optionally routes the /voice_agents call directly to -// a regional data-plane host (bypassing the public Foundry APIM, whose voice -// route may not yet be rolled out). When unset, default endpoint routing is used. +// voiceOverriddenHostEnvKey optionally routes prompt voice agent calls directly +// to a regional data-plane host (bypassing the public Foundry APIM when needed). +// When unset, default endpoint routing is used. // //nolint:gosec // env var key name, not a credential const voiceOverriddenHostEnvKey = "AZURE_VOICE_OVERRIDDEN_HOST" -// voiceAgentAPIEnvKey controls which voice deployment API azd uses. It defaults -// to the legacy /voice_agents path while the unified API rolls out regionally. -// Supported values: -// - legacy: POST /voice_agents with object-shaped audio.output.voice -// - unified: POST /agents or /agents/{name} with object-shaped audio.output.voice -// - unified-flat: POST /agents or /agents/{name} with flat audio.output.voice -// -//nolint:gosec // env var key name, not a credential -const voiceAgentAPIEnvKey = "AZURE_VOICE_AGENT_API" - -type voiceAgentAPIMode string - -const ( - voiceAgentAPIModeLegacy voiceAgentAPIMode = "legacy" - voiceAgentAPIModeUnified voiceAgentAPIMode = "unified" - voiceAgentAPIModeUnifiedFlat voiceAgentAPIMode = "unified-flat" -) - -func resolveVoiceAgentAPIMode(azdEnv map[string]string) (voiceAgentAPIMode, error) { - mode := strings.TrimSpace(azdEnv[voiceAgentAPIEnvKey]) - if mode == "" { - mode = strings.TrimSpace(os.Getenv(voiceAgentAPIEnvKey)) - } - mode = strings.ToLower(strings.ReplaceAll(mode, "_", "-")) - if mode == "" { - return voiceAgentAPIModeLegacy, nil - } - switch voiceAgentAPIMode(mode) { - case voiceAgentAPIModeLegacy, voiceAgentAPIModeUnified, voiceAgentAPIModeUnifiedFlat: - return voiceAgentAPIMode(mode), nil - default: - return "", fmt.Errorf( - "%s must be one of %q, %q, or %q", - voiceAgentAPIEnvKey, voiceAgentAPIModeLegacy, voiceAgentAPIModeUnified, voiceAgentAPIModeUnifiedFlat, - ) - } -} - // deployVoiceAgent deploys a declarative (managed) voice agent (kind: -// prompt-voice) to the Foundry service. The legacy /voice_agents API remains -// the default while unified /agents rolls out regionally; AZURE_VOICE_AGENT_API -// can opt into unified modes for regression and TiP validation. This method is -// intentionally isolated from the container deploy path so the two contracts -// never entangle. +// prompt-voice) to the Foundry service through the unified /agents API. This +// method is intentionally isolated from the container deploy path so the two +// contracts never entangle. func (p *AgentServiceTargetProvider) deployVoiceAgent( ctx context.Context, serviceConfig *azdext.ServiceConfig, @@ -2215,21 +2175,7 @@ func (p *AgentServiceTargetProvider) deployVoiceAgent( ) (*azdext.ServiceDeployResult, error) { progress("Deploying voice agent") - apiMode, err := resolveVoiceAgentAPIMode(azdEnv) - if err != nil { - return nil, exterrors.Validation( - exterrors.CodeInvalidParameter, - err.Error(), - fmt.Sprintf("set %s to legacy, unified, or unified-flat", voiceAgentAPIEnvKey), - ) - } - - var request *agent_api.CreateAgentRequest - if apiMode == voiceAgentAPIModeUnifiedFlat { - request, err = agent_yaml.CreateVoiceAgentAPIRequestFlat(va) - } else { - request, err = agent_yaml.CreateVoiceAgentAPIRequest(va) - } + request, err := agent_yaml.CreateVoiceAgentAPIRequestFlat(va) if err != nil { return nil, exterrors.Validation( exterrors.CodeInvalidAgentManifest, @@ -2251,13 +2197,13 @@ func (p *AgentServiceTargetProvider) deployVoiceAgent( agentClient := agent_api.NewAgentClient(projectEndpoint, p.credential) serviceKey := p.getServiceKey(serviceConfig.Name) - agentObject, deployOp, err := p.deployVoiceAgentWithMode( - ctx, agentClient, request, apiMode, azdEnv, progress, + agentObject, deployOp, err := p.deployVoiceAgentUnified( + ctx, agentClient, request, azdEnv, progress, ) if err != nil { return nil, exterrors.ServiceFromAzure(err, deployOp) } - if err := validateVoiceAgentDeployResponse(agentObject, apiMode); err != nil { + if err := validateVoiceAgentDeployResponse(agentObject); err != nil { return nil, err } @@ -2265,12 +2211,9 @@ func (p *AgentServiceTargetProvider) deployVoiceAgent( // Persist NAME first and ENDPOINT last. ENDPOINT is used as the voice deploy // completion marker by other commands, so avoid writing it before NAME. - baseEndpoint := voiceAgentEndpoint(projectEndpoint, agentObject.Name, apiMode) + baseEndpoint := buildVoiceWSProtocolURL(projectEndpoint, agentObject.Name) versionKey := fmt.Sprintf("AGENT_%s_VERSION", serviceKey) - versionValue := "" - if apiMode != voiceAgentAPIModeLegacy { - versionValue = agentObject.Versions.Latest.Version - } + versionValue := agentObject.Versions.Latest.Version for _, envVar := range []struct{ key, value string }{ {fmt.Sprintf("AGENT_%s_NAME", serviceKey), agentObject.Name}, {versionKey, versionValue}, @@ -2299,34 +2242,27 @@ func (p *AgentServiceTargetProvider) deployVoiceAgent( return &azdext.ServiceDeployResult{Artifacts: artifacts}, nil } -func validateVoiceAgentDeployResponse(agentObject *agent_api.AgentObject, apiMode voiceAgentAPIMode) error { +func validateVoiceAgentDeployResponse(agentObject *agent_api.AgentObject) error { if agentObject == nil { return fmt.Errorf("malformed voice agent service response: missing agent object") } if strings.TrimSpace(agentObject.Name) == "" { return fmt.Errorf("malformed voice agent service response: missing agent name") } - if apiMode != voiceAgentAPIModeLegacy && strings.TrimSpace(agentObject.Versions.Latest.Version) == "" { + if strings.TrimSpace(agentObject.Versions.Latest.Version) == "" { return fmt.Errorf("malformed voice agent service response: missing latest agent version") } return nil } -func (p *AgentServiceTargetProvider) deployVoiceAgentWithMode( +func (p *AgentServiceTargetProvider) deployVoiceAgentUnified( ctx context.Context, agentClient *agent_api.AgentClient, request *agent_api.CreateAgentRequest, - apiMode voiceAgentAPIMode, azdEnv map[string]string, progress azdext.ProgressReporter, ) (*agent_api.AgentObject, string, error) { overriddenHost := azdEnv[voiceOverriddenHostEnvKey] - if apiMode == voiceAgentAPIModeLegacy { - progress("Creating voice agent using legacy API") - agentObject, err := agentClient.CreateVoiceAgent(ctx, request, agent_api.AgentEndpointAPIVersion, overriddenHost) - return agentObject, exterrors.OpCreateAgent, err - } - remoteAgent, getErr := agentClient.GetVoiceAgentUnified( ctx, request.Name, agent_api.AgentEndpointAPIVersion, overriddenHost, ) @@ -2360,14 +2296,6 @@ func shouldUpdateVoiceAgent(remoteAgent *agent_api.AgentObject, getErr error) (b return false, getErr } -func voiceAgentEndpoint(projectEndpoint string, agentName string, apiMode voiceAgentAPIMode) string { - trimmedEndpoint := strings.TrimRight(projectEndpoint, "/") - if apiMode == voiceAgentAPIModeLegacy { - return fmt.Sprintf("%s/voice_agents/%s", trimmedEndpoint, agentName) - } - return buildVoiceWSProtocolURL(trimmedEndpoint, agentName) -} - // packageCodeDeploy creates a ZIP archive of the agent source code, writes it to a temp file, // and computes its SHA-256. Returns the temp file path and SHA-256 hex string. func (p *AgentServiceTargetProvider) packageCodeDeploy(ctx context.Context, serviceConfig *azdext.ServiceConfig) (string, string, error) { diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go index bd4d87fa290..925c9de1256 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go @@ -669,44 +669,6 @@ func TestAdoptServiceConfigIgnoresNilAndKeepsResolvedState(t *testing.T) { require.False(t, provider.serviceConfigResolved) } -func TestResolveVoiceAgentAPIMode_DefaultsToLegacy(t *testing.T) { - t.Setenv(voiceAgentAPIEnvKey, "") - mode, err := resolveVoiceAgentAPIMode(map[string]string{}) - require.NoError(t, err) - require.Equal(t, voiceAgentAPIModeLegacy, mode) -} - -func TestResolveVoiceAgentAPIMode_EnvValues(t *testing.T) { - t.Setenv(voiceAgentAPIEnvKey, "unified_flat") - mode, err := resolveVoiceAgentAPIMode(map[string]string{}) - require.NoError(t, err) - require.Equal(t, voiceAgentAPIModeUnifiedFlat, mode) - - mode, err = resolveVoiceAgentAPIMode(map[string]string{voiceAgentAPIEnvKey: "unified"}) - require.NoError(t, err) - require.Equal(t, voiceAgentAPIModeUnified, mode) -} - -func TestResolveVoiceAgentAPIMode_Invalid(t *testing.T) { - _, err := resolveVoiceAgentAPIMode(map[string]string{voiceAgentAPIEnvKey: "future"}) - require.Error(t, err) - require.Contains(t, err.Error(), voiceAgentAPIEnvKey) -} - -func TestVoiceAgentEndpoint_ByMode(t *testing.T) { - projectEndpoint := "https://proj.services.ai.azure.com/api/projects/p/" - require.Equal( - t, - "https://proj.services.ai.azure.com/api/projects/p/voice_agents/my-agent", - voiceAgentEndpoint(projectEndpoint, "my-agent", voiceAgentAPIModeLegacy), - ) - require.Equal( - t, - "wss://proj.services.ai.azure.com/api/projects/p/agents/my-agent/endpoint/protocols/voice?api-version=v1", - voiceAgentEndpoint(projectEndpoint, "my-agent", voiceAgentAPIModeUnified), - ) -} - func TestBuildVoiceWSProtocolURL(t *testing.T) { got := buildVoiceWSProtocolURL( "https://acct.services.ai.azure.com/api/projects/proj/", @@ -720,31 +682,20 @@ func TestBuildVoiceWSProtocolURL(t *testing.T) { } func TestValidateVoiceAgentDeployResponse(t *testing.T) { - t.Run("legacy requires name only", func(t *testing.T) { - err := validateVoiceAgentDeployResponse( - &agent_api.AgentObject{Name: "voice-agent"}, - voiceAgentAPIModeLegacy, - ) - require.NoError(t, err) - }) - - t.Run("unified requires latest version", func(t *testing.T) { + t.Run("requires name and latest version", func(t *testing.T) { agent := &agent_api.AgentObject{Name: "voice-agent"} agent.Versions.Latest.Version = "1" - err := validateVoiceAgentDeployResponse(agent, voiceAgentAPIModeUnifiedFlat) + err := validateVoiceAgentDeployResponse(agent) require.NoError(t, err) }) t.Run("missing name rejected", func(t *testing.T) { - err := validateVoiceAgentDeployResponse(&agent_api.AgentObject{}, voiceAgentAPIModeLegacy) + err := validateVoiceAgentDeployResponse(&agent_api.AgentObject{}) require.ErrorContains(t, err, "missing agent name") }) - t.Run("unified missing version rejected", func(t *testing.T) { - err := validateVoiceAgentDeployResponse( - &agent_api.AgentObject{Name: "voice-agent"}, - voiceAgentAPIModeUnified, - ) + t.Run("missing version rejected", func(t *testing.T) { + err := validateVoiceAgentDeployResponse(&agent_api.AgentObject{Name: "voice-agent"}) require.ErrorContains(t, err, "missing latest agent version") }) } From 3c733352b7e757e0cc0336a671cba2402f9d458a Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Mon, 24 Aug 2026 16:48:12 +0800 Subject: [PATCH 18/38] docs(agents): clarify voice deploy marker --- .../azure.ai.agents/internal/cmd/nextstep/state.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/nextstep/state.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/nextstep/state.go index afafade125c..768e776e2f9 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/nextstep/state.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/nextstep/state.go @@ -826,9 +826,9 @@ func isDeployed( return value != "" } - // Voice deploys use the base ENDPOINT env var as the completion marker. The - // legacy voice API does not produce AGENT__VERSION, and unified voice - // deploys write VERSION before ENDPOINT to keep ENDPOINT as the final marker. + // Voice deploys use the base ENDPOINT env var as the completion marker. + // Unified voice deploys write VERSION before ENDPOINT to keep ENDPOINT as the + // final marker. // Require ENDPOINT for voice even when VERSION is present, otherwise a partial // env write could be reported as deployed before the callable endpoint was // persisted. Gate this on the service's actual declared kind: a hosted agent From 62ef41e3f96a700b8ea89d604aa899a8fa550cde Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Mon, 24 Aug 2026 17:14:34 +0800 Subject: [PATCH 19/38] docs(agents): clarify unified voice version state Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: c76c1120-2c86-448b-8ed3-308d53db7dac --- .../cmd/doctor/checks_agent_status.go | 7 ++++--- .../internal/cmd/nextstep/state.go | 6 +++--- .../internal/project/service_target_agent.go | 19 ++++++++++--------- .../project/service_target_agent_test.go | 18 +++++++++--------- 4 files changed, 26 insertions(+), 24 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/doctor/checks_agent_status.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/doctor/checks_agent_status.go index cdffffe264d..840792460b8 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/doctor/checks_agent_status.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/doctor/checks_agent_status.go @@ -704,9 +704,10 @@ func readAgentServices(prior []Result) []string { // filterHostedAgentServices removes prompt-voice services from the hosted-agent // status probe. The remote.agent-status check targets hosted agent versions -// (`/agents/{name}/versions/{version}`) and relies on AGENT__VERSION; a -// prompt-voice deploy writes NAME+ENDPOINT only and should be covered by a -// voice-specific doctor check in a future PR. +// (`/agents/{name}/versions/{version}`). Unified prompt-voice deploys record a +// VERSION, but voice readiness follows its WebSocket endpoint rather than the +// hosted-agent version lifecycle and should be covered by a voice-specific +// doctor check in a future PR. func filterHostedAgentServices(ctx context.Context, azdClient *azdext.AzdClient, services []string) []string { if len(services) == 0 || azdClient == nil { return services diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/nextstep/state.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/nextstep/state.go index 768e776e2f9..e06719e3330 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/nextstep/state.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/nextstep/state.go @@ -36,9 +36,9 @@ const ( agentVersionVarFormat = "AGENT_%s_VERSION" // agentEndpointVarFormat is the base endpoint env-var written for every - // deployed agent. Voice agents (kind: prompt-voice) are created - // synchronously with no agent-version object, so this base endpoint is the - // only deployment marker they set — isDeployed falls back to it. + // deployed agent. Voice agents (kind: prompt-voice) use it as the deploy + // completion marker. Unified voice deploys also set VERSION, while legacy + // voice environments may have only this endpoint marker. agentEndpointVarFormat = "AGENT_%s_ENDPOINT" // projectEndpointVar is the env-var that carries the Foundry project diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go index 51f590ce0cf..39379969eea 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go @@ -551,15 +551,16 @@ func (p *AgentServiceTargetProvider) Endpoints( agentVersionKey := fmt.Sprintf("AGENT_%s_VERSION", serviceKey) agentEndpointKey := fmt.Sprintf("AGENT_%s_ENDPOINT", serviceKey) - // Voice agents (kind: prompt-voice) are created synchronously with no - // agent-version object and no per-protocol endpoints; they record only NAME - // and a base ENDPOINT. Gate the base-endpoint fallback on the service's - // actual declared kind (resolved via the shared agentkind lookup, so this - // agrees with the deploy path and next-step reader) rather than on the - // env-var shape: a hosted agent whose deploy partially failed (or whose vars - // were cleaned up) can also present an empty VERSION with a lingering - // ENDPOINT, and for that case we must still surface the actionable - // CodeMissingAgentEnvVars error below. Kind resolution is best-effort here: + // Voice agents (kind: prompt-voice) use the base ENDPOINT as their callable + // endpoint and deploy completion marker. Unified deploys also record VERSION, + // while environments created by the legacy voice API may have only + // NAME+ENDPOINT. Gate the base-endpoint path on the service's actual declared + // kind (resolved via the shared agentkind lookup, so this agrees with the + // deploy path and next-step reader) rather than on the env-var shape: a hosted + // agent whose deploy partially failed (or whose vars were cleaned up) can also + // present an empty VERSION with a lingering ENDPOINT, and for that case we + // must still surface the actionable CodeMissingAgentEnvVars error below. + // Kind resolution is best-effort here: // an error (or non-voice result) simply falls through to the hosted guard, so // hosted services keep their prior behavior on a path that never resolved // config before. diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go index 925c9de1256..a3c0af5af9f 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go @@ -2810,9 +2810,9 @@ func newEndpointsTestClient( // TestEndpoints_VoiceManifestOnDisk_ResolvesProjectRoot covers the fresh-process // case where Endpoints runs without ensureDeployContext having populated // p.projectPath. A legacy-shape prompt-voice service (kind only on disk, no -// inline kind) records NAME+ENDPOINT but no VERSION; Endpoints must resolve the -// project root itself so agentkind classifies it as voice and returns the base -// endpoint instead of the missing-VERSION error. +// inline kind) may retain NAME+ENDPOINT without VERSION from an earlier deploy; +// Endpoints must resolve the project root itself so agentkind classifies it as +// voice and returns the base endpoint instead of the missing-VERSION error. func TestEndpoints_VoiceManifestOnDisk_ResolvesProjectRoot(t *testing.T) { t.Parallel() @@ -2830,7 +2830,7 @@ func TestEndpoints_VoiceManifestOnDisk_ResolvesProjectRoot(t *testing.T) { "FOUNDRY_PROJECT_ENDPOINT": "https://proj.services.ai.azure.com", "AGENT_VOICE_NAME": "my-voice", "AGENT_VOICE_ENDPOINT": endpoint, - // deliberately no AGENT_VOICE_VERSION: voice agents have no version. + // Deliberately model a legacy persisted environment with no VERSION. }) // Fresh process: projectPath/agentDefinitionPath are empty, exactly as they @@ -2848,10 +2848,10 @@ func TestEndpoints_VoiceManifestOnDisk_ResolvesProjectRoot(t *testing.T) { // TestEndpoints_VoiceAgentDefinitionPathOverride covers the fresh-process case // where a voice manifest is supplied via the AGENT_DEFINITION_PATH override. -// Deploy follows the override and writes NAME+ENDPOINT but no VERSION; Endpoints -// runs without ensureDeployContext (so p.agentDefinitionPath is empty) and must -// read the process override to classify the service as voice, rather than -// classifying the (kind-less) service entry and returning missing-VERSION. +// Endpoints runs without ensureDeployContext (so p.agentDefinitionPath is empty) +// and must read the process override to classify a legacy persisted +// NAME+ENDPOINT environment as voice, rather than classifying the (kind-less) +// service entry and returning missing-VERSION. func TestEndpoints_VoiceAgentDefinitionPathOverride(t *testing.T) { projectRoot := t.TempDir() overridePath := filepath.Join(projectRoot, "custom-voice.yaml") @@ -2867,7 +2867,7 @@ func TestEndpoints_VoiceAgentDefinitionPathOverride(t *testing.T) { "FOUNDRY_PROJECT_ENDPOINT": "https://proj.services.ai.azure.com", "AGENT_VOICE_NAME": "my-voice", "AGENT_VOICE_ENDPOINT": endpoint, - // no AGENT_VOICE_VERSION: voice agents have no version. + // Deliberately model a legacy persisted environment with no VERSION. }) // Fresh process: the service entry carries no kind; only the override does. From 31a2860fb6f723542e5513bd6b7706abca68f4c1 Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Mon, 24 Aug 2026 19:44:57 +0800 Subject: [PATCH 20/38] docs(agents): clarify unified voice default --- .../azure.ai.agents/internal/pkg/agents/agent_api/operations.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations.go index ce9aff37322..cfa24decde8 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations.go @@ -260,7 +260,7 @@ func (c *AgentClient) GetVoiceAgentUnified( } // CreateVoiceAgentUnified creates a voice agent through the unified /agents -// collection. This path is opt-in while regional rollout is still in progress. +// collection. Prompt voice deploys use this path by default. func (c *AgentClient) CreateVoiceAgentUnified( ctx context.Context, request *CreateAgentRequest, From 5748dbdda2193b67ca601d4fe8935a99b59a9693 Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Tue, 25 Aug 2026 08:59:59 +0800 Subject: [PATCH 21/38] refactor(agents): replace voice API surface --- .../internal/pkg/agents/agent_api/models.go | 36 +++------------- .../pkg/agents/agent_api/operations.go | 12 +++--- .../pkg/agents/agent_api/operations_test.go | 12 +++--- .../internal/pkg/agents/agent_yaml/map.go | 14 ++---- .../pkg/agents/agent_yaml/map_voice_test.go | 43 +++++++++++-------- .../internal/project/service_target_agent.go | 17 ++++---- 6 files changed, 54 insertions(+), 80 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go index 946ac70951d..86b452662d3 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go @@ -391,16 +391,10 @@ type VoiceConfig struct { Name string `json:"name"` } -// VoiceOutputConfig is the output (agent -> caller) audio configuration. +// VoiceOutputConfig is the output (agent -> caller) audio configuration for the +// unified /agents voice API. The voice name is a string and provider details are +// sibling fields. type VoiceOutputConfig struct { - Format *VoiceAudioFormat `json:"format,omitempty"` - Voice *VoiceConfig `json:"voice,omitempty"` -} - -// VoiceOutputConfigFlat is the newer Voice Live output shape used by the -// unified /agents voice API in TiP. Older regions still accept/return the -// object-shaped VoiceOutputConfig above. -type VoiceOutputConfigFlat struct { Format *VoiceAudioFormat `json:"format,omitempty"` Voice string `json:"voice,omitempty"` VoiceType string `json:"voice_type,omitempty"` @@ -413,15 +407,8 @@ type VoiceAudioConfig struct { Output *VoiceOutputConfig `json:"output,omitempty"` } -// VoiceAudioConfigFlat bundles voice audio config with the flat output shape. -type VoiceAudioConfigFlat struct { - Input *VoiceInputConfig `json:"input,omitempty"` - Output *VoiceOutputConfigFlat `json:"output,omitempty"` -} - -// VoiceAgentDefinition is retained for compatibility with object-shaped voice -// definitions returned by older services. New prompt voice deploys use -// VoiceAgentDefinitionFlat. +// VoiceAgentDefinition is the data-plane definition body for a declarative +// prompt voice agent. Its Kind is always AgentKindVoice ("voice"). type VoiceAgentDefinition struct { AgentDefinition ModelType VoiceModelType `json:"model_type"` @@ -432,19 +419,6 @@ type VoiceAgentDefinition struct { Store *bool `json:"store,omitempty"` } -// VoiceAgentDefinitionFlat is the voice definition shape aligned with the -// unified /agents TiP API, where audio.output.voice is a string and the voice -// provider details are sibling fields. -type VoiceAgentDefinitionFlat struct { - AgentDefinition - ModelType VoiceModelType `json:"model_type"` - Model string `json:"model"` - Instructions string `json:"instructions,omitempty"` - Audio *VoiceAudioConfigFlat `json:"audio,omitempty"` - OutputModalities []string `json:"output_modalities,omitempty"` - Store *bool `json:"store,omitempty"` -} - // CreateAgentVersionRequest represents a request to create an agent version type CreateAgentVersionRequest struct { Description *string `json:"description,omitempty"` diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations.go index cfa24decde8..6670f8dc455 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations.go @@ -216,10 +216,10 @@ func (c *AgentClient) doVoiceJSONAgentRequest( return &agent, nil } -// GetVoiceAgentUnified retrieves a voice agent through the unified /agents +// GetVoiceAgent retrieves a voice agent through the unified /agents // endpoint with the voice preview opt-in header. Use this instead of GetAgent // when deciding whether to create or update a prompt voice agent. -func (c *AgentClient) GetVoiceAgentUnified( +func (c *AgentClient) GetVoiceAgent( ctx context.Context, agentName string, apiVersion string, @@ -259,9 +259,9 @@ func (c *AgentClient) GetVoiceAgentUnified( return &agent, nil } -// CreateVoiceAgentUnified creates a voice agent through the unified /agents +// CreateVoiceAgent creates a voice agent through the unified /agents // collection. Prompt voice deploys use this path by default. -func (c *AgentClient) CreateVoiceAgentUnified( +func (c *AgentClient) CreateVoiceAgent( ctx context.Context, request *CreateAgentRequest, apiVersion string, @@ -271,9 +271,9 @@ func (c *AgentClient) CreateVoiceAgentUnified( return c.doVoiceJSONAgentRequest(ctx, http.MethodPost, url, request, overriddenHost) } -// UpdateVoiceAgentUnified creates a new version for an existing voice agent +// UpdateVoiceAgent creates a new version for an existing voice agent // through the unified /agents/{name} endpoint. -func (c *AgentClient) UpdateVoiceAgentUnified( +func (c *AgentClient) UpdateVoiceAgent( ctx context.Context, agentName string, request *UpdateAgentRequest, diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations_test.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations_test.go index d47e3a5dee1..58e84f0a573 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations_test.go @@ -919,11 +919,11 @@ func TestDownloadAgentCode_ReturnsErrorOnNon200(t *testing.T) { require.Error(t, err) } -func TestCreateVoiceAgentUnified_PostsToAgentsWithPreviewHeader(t *testing.T) { +func TestCreateVoiceAgent_PostsToAgentsWithPreviewHeader(t *testing.T) { body := `{"object":"agent","id":"va-1","name":"my-voice","versions":{"latest":{}}}` client, transport := newCaptureClient(http.StatusOK, body) - agent, err := client.CreateVoiceAgentUnified( + agent, err := client.CreateVoiceAgent( t.Context(), &CreateAgentRequest{Name: "my-voice"}, AgentEndpointAPIVersion, @@ -940,11 +940,11 @@ func TestCreateVoiceAgentUnified_PostsToAgentsWithPreviewHeader(t *testing.T) { require.Equal(t, voiceAgentsPreviewFeature, req.Header.Get("Foundry-Features")) } -func TestGetVoiceAgentUnified_GetsNamedAgentWithPreviewHeader(t *testing.T) { +func TestGetVoiceAgent_GetsNamedAgentWithPreviewHeader(t *testing.T) { body := `{"object":"agent","id":"va-1","name":"my-voice","versions":{"latest":{"version":"3"}}}` client, transport := newCaptureClient(http.StatusOK, body) - agent, err := client.GetVoiceAgentUnified( + agent, err := client.GetVoiceAgent( t.Context(), "my-voice", AgentEndpointAPIVersion, @@ -963,11 +963,11 @@ func TestGetVoiceAgentUnified_GetsNamedAgentWithPreviewHeader(t *testing.T) { require.Equal(t, "regional.hyena.example.com", req.Header.Get("x-ms-overridden-host")) } -func TestUpdateVoiceAgentUnified_PostsToNamedAgentWithPreviewHeader(t *testing.T) { +func TestUpdateVoiceAgent_PostsToNamedAgentWithPreviewHeader(t *testing.T) { body := `{"object":"agent","id":"va-1","name":"my-voice","versions":{"latest":{"version":"2"}}}` client, transport := newCaptureClient(http.StatusOK, body) - agent, err := client.UpdateVoiceAgentUnified( + agent, err := client.UpdateVoiceAgent( t.Context(), "my-voice", &UpdateAgentRequest{}, diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go index b2c9ae5cb9d..3225ef51fb2 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go @@ -611,14 +611,6 @@ func CreateVoiceAgentAPIRequest(voiceAgent VoiceAgent) (*agent_api.CreateAgentRe return createVoiceAgentAPIRequest(voiceAgent) } -// CreateVoiceAgentAPIRequestFlat is kept as a compatibility wrapper for tests -// and callers that explicitly selected the transition name while the unified API -// work was in flight. Prompt voice requests now always use the unified flat -// output shape. -func CreateVoiceAgentAPIRequestFlat(voiceAgent VoiceAgent) (*agent_api.CreateAgentRequest, error) { - return createVoiceAgentAPIRequest(voiceAgent) -} - func createVoiceAgentAPIRequest(voiceAgent VoiceAgent) (*agent_api.CreateAgentRequest, error) { modelID := "" if voiceAgent.Model != nil { @@ -659,7 +651,7 @@ func createVoiceAgentAPIRequest(voiceAgent VoiceAgent) (*agent_api.CreateAgentRe Transcription: &agent_api.VoiceTranscription{Model: defaultVoiceInputTranscriptionModel}, } voiceConfig := buildVoiceConfig(voiceName) - voiceDef := agent_api.VoiceAgentDefinitionFlat{ + voiceDef := agent_api.VoiceAgentDefinition{ AgentDefinition: agent_api.AgentDefinition{ // Translate authoring kind prompt-voice -> service kind voice. Kind: agent_api.AgentKindVoice, @@ -667,9 +659,9 @@ func createVoiceAgentAPIRequest(voiceAgent VoiceAgent) (*agent_api.CreateAgentRe ModelType: modelType, Model: modelID, Instructions: instructions, - Audio: &agent_api.VoiceAudioConfigFlat{ + Audio: &agent_api.VoiceAudioConfig{ Input: input, - Output: &agent_api.VoiceOutputConfigFlat{ + Output: &agent_api.VoiceOutputConfig{ Format: audioFormat, Voice: voiceConfig.Name, VoiceType: flatVoiceType(voiceConfig), diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go index 4b6e78ae065..5d51ac12822 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go @@ -90,9 +90,9 @@ func TestCreateVoiceAgentAPIRequest_Defaults(t *testing.T) { t.Errorf("Name = %q", req.Name) } - def, ok := req.Definition.(agent_api.VoiceAgentDefinitionFlat) + def, ok := req.Definition.(agent_api.VoiceAgentDefinition) if !ok { - t.Fatalf("expected VoiceAgentDefinitionFlat, got %T", req.Definition) + t.Fatalf("expected VoiceAgentDefinition, got %T", req.Definition) } // Authoring kind prompt-voice is translated to service kind voice. @@ -160,7 +160,7 @@ func TestCreateVoiceAgentAPIRequest_Overrides(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - def := req.Definition.(agent_api.VoiceAgentDefinitionFlat) + def := req.Definition.(agent_api.VoiceAgentDefinition) if def.Instructions != instructions { t.Errorf("Instructions = %q", def.Instructions) } @@ -173,7 +173,7 @@ func TestCreateVoiceAgentAPIRequest_Overrides(t *testing.T) { } } -func TestCreateVoiceAgentAPIRequestFlat_UsesFlatOutputShape(t *testing.T) { +func TestCreateVoiceAgentAPIRequest_UsesFlatOutputShape(t *testing.T) { t.Parallel() voice := "alloy" agent := VoiceAgent{ @@ -182,11 +182,11 @@ func TestCreateVoiceAgentAPIRequestFlat_UsesFlatOutputShape(t *testing.T) { Voice: &voice, } - req, err := CreateVoiceAgentAPIRequestFlat(agent) + req, err := CreateVoiceAgentAPIRequest(agent) if err != nil { t.Fatalf("unexpected error: %v", err) } - def := req.Definition.(agent_api.VoiceAgentDefinitionFlat) + def := req.Definition.(agent_api.VoiceAgentDefinition) if def.Audio.Output.Voice != "alloy" { t.Errorf("Voice = %q, want alloy", def.Audio.Output.Voice) } @@ -198,7 +198,7 @@ func TestCreateVoiceAgentAPIRequestFlat_UsesFlatOutputShape(t *testing.T) { } } -func TestCreateVoiceAgentAPIRequestFlat_AzureVoiceLocale(t *testing.T) { +func TestCreateVoiceAgentAPIRequest_AzureVoiceLocale(t *testing.T) { t.Parallel() voice := "en-US-Ava:DragonHDLatestNeural" agent := VoiceAgent{ @@ -207,11 +207,11 @@ func TestCreateVoiceAgentAPIRequestFlat_AzureVoiceLocale(t *testing.T) { Voice: &voice, } - req, err := CreateVoiceAgentAPIRequestFlat(agent) + req, err := CreateVoiceAgentAPIRequest(agent) if err != nil { t.Fatalf("unexpected error: %v", err) } - def := req.Definition.(agent_api.VoiceAgentDefinitionFlat) + def := req.Definition.(agent_api.VoiceAgentDefinition) if def.Audio.Output.Voice != voice { t.Errorf("Voice = %q, want %q", def.Audio.Output.Voice, voice) } @@ -223,7 +223,7 @@ func TestCreateVoiceAgentAPIRequestFlat_AzureVoiceLocale(t *testing.T) { } } -func TestCreateVoiceAgentAPIRequestFlat_MarshalWireShape(t *testing.T) { +func TestCreateVoiceAgentAPIRequest_MarshalWireShape(t *testing.T) { t.Parallel() voice := "en-US-Ava:DragonHDLatestNeural" agent := VoiceAgent{ @@ -232,7 +232,7 @@ func TestCreateVoiceAgentAPIRequestFlat_MarshalWireShape(t *testing.T) { Voice: &voice, } - req, err := CreateVoiceAgentAPIRequestFlat(agent) + req, err := CreateVoiceAgentAPIRequest(agent) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -245,9 +245,18 @@ func TestCreateVoiceAgentAPIRequestFlat_MarshalWireShape(t *testing.T) { if err := json.Unmarshal(payload, &wire); err != nil { t.Fatalf("unmarshal payload: %v", err) } - definition := wire["definition"].(map[string]any) - audio := definition["audio"].(map[string]any) - output := audio["output"].(map[string]any) + definition, ok := wire["definition"].(map[string]any) + if !ok { + t.Fatalf("definition = %#v, want object", wire["definition"]) + } + audio, ok := definition["audio"].(map[string]any) + if !ok { + t.Fatalf("definition.audio = %#v, want object", definition["audio"]) + } + output, ok := audio["output"].(map[string]any) + if !ok { + t.Fatalf("definition.audio.output = %#v, want object", audio["output"]) + } if got, ok := output["voice"].(string); !ok || got != voice { t.Fatalf("audio.output.voice = %#v, want string %q", output["voice"], voice) @@ -276,7 +285,7 @@ func TestCreateVoiceAgentAPIRequest_ExplicitManaged(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - if req.Definition.(agent_api.VoiceAgentDefinitionFlat).ModelType != agent_api.VoiceModelTypeManaged { + if req.Definition.(agent_api.VoiceAgentDefinition).ModelType != agent_api.VoiceModelTypeManaged { t.Errorf("ModelType not managed") } } @@ -304,7 +313,7 @@ func TestCreateVoiceAgentAPIRequest_TrimsModelID(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - def := req.Definition.(agent_api.VoiceAgentDefinitionFlat) + def := req.Definition.(agent_api.VoiceAgentDefinition) if def.Model != "my-realtime-deployment" { t.Errorf("Model = %q, want trimmed model id", def.Model) } @@ -334,7 +343,7 @@ func TestCreateVoiceAgentAPIRequest_SelfDeployedMapped(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - def := req.Definition.(agent_api.VoiceAgentDefinitionFlat) + def := req.Definition.(agent_api.VoiceAgentDefinition) if def.ModelType != agent_api.VoiceModelTypeSelfDeployed { t.Errorf("ModelType = %q, want self_deployed", def.ModelType) } diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go index 39379969eea..4d2fb1f1ae4 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go @@ -552,9 +552,8 @@ func (p *AgentServiceTargetProvider) Endpoints( agentEndpointKey := fmt.Sprintf("AGENT_%s_ENDPOINT", serviceKey) // Voice agents (kind: prompt-voice) use the base ENDPOINT as their callable - // endpoint and deploy completion marker. Unified deploys also record VERSION, - // while environments created by the legacy voice API may have only - // NAME+ENDPOINT. Gate the base-endpoint path on the service's actual declared + // endpoint and deploy completion marker, and unified deploys also record + // VERSION. Gate the base-endpoint path on the service's actual declared // kind (resolved via the shared agentkind lookup, so this agrees with the // deploy path and next-step reader) rather than on the env-var shape: a hosted // agent whose deploy partially failed (or whose vars were cleaned up) can also @@ -2176,7 +2175,7 @@ func (p *AgentServiceTargetProvider) deployVoiceAgent( ) (*azdext.ServiceDeployResult, error) { progress("Deploying voice agent") - request, err := agent_yaml.CreateVoiceAgentAPIRequestFlat(va) + request, err := agent_yaml.CreateVoiceAgentAPIRequest(va) if err != nil { return nil, exterrors.Validation( exterrors.CodeInvalidAgentManifest, @@ -2198,7 +2197,7 @@ func (p *AgentServiceTargetProvider) deployVoiceAgent( agentClient := agent_api.NewAgentClient(projectEndpoint, p.credential) serviceKey := p.getServiceKey(serviceConfig.Name) - agentObject, deployOp, err := p.deployVoiceAgentUnified( + agentObject, deployOp, err := p.deployVoiceAgentRemote( ctx, agentClient, request, azdEnv, progress, ) if err != nil { @@ -2256,7 +2255,7 @@ func validateVoiceAgentDeployResponse(agentObject *agent_api.AgentObject) error return nil } -func (p *AgentServiceTargetProvider) deployVoiceAgentUnified( +func (p *AgentServiceTargetProvider) deployVoiceAgentRemote( ctx context.Context, agentClient *agent_api.AgentClient, request *agent_api.CreateAgentRequest, @@ -2264,7 +2263,7 @@ func (p *AgentServiceTargetProvider) deployVoiceAgentUnified( progress azdext.ProgressReporter, ) (*agent_api.AgentObject, string, error) { overriddenHost := azdEnv[voiceOverriddenHostEnvKey] - remoteAgent, getErr := agentClient.GetVoiceAgentUnified( + remoteAgent, getErr := agentClient.GetVoiceAgent( ctx, request.Name, agent_api.AgentEndpointAPIVersion, overriddenHost, ) shouldUpdate, decisionErr := shouldUpdateVoiceAgent(remoteAgent, getErr) @@ -2276,14 +2275,14 @@ func (p *AgentServiceTargetProvider) deployVoiceAgentUnified( updateRequest := &agent_api.UpdateAgentRequest{ CreateAgentVersionRequest: request.CreateAgentVersionRequest, } - agentObject, err := agentClient.UpdateVoiceAgentUnified( + agentObject, err := agentClient.UpdateVoiceAgent( ctx, request.Name, updateRequest, agent_api.AgentEndpointAPIVersion, overriddenHost, ) return agentObject, exterrors.OpUpdateAgent, err } progress("Creating voice agent using unified API") - agentObject, err := agentClient.CreateVoiceAgentUnified(ctx, request, agent_api.AgentEndpointAPIVersion, overriddenHost) + agentObject, err := agentClient.CreateVoiceAgent(ctx, request, agent_api.AgentEndpointAPIVersion, overriddenHost) return agentObject, exterrors.OpCreateAgent, err } From b9ad10ece966fede49480302c5ad2abec3dd6bf7 Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Tue, 25 Aug 2026 09:15:34 +0800 Subject: [PATCH 22/38] docs(agents): clean voice API migration wording --- .../internal/cmd/doctor/checks_agent_status.go | 2 +- .../azure.ai.agents/internal/cmd/nextstep/state.go | 4 ++-- .../internal/cmd/nextstep/state_test.go | 6 +++--- .../internal/pkg/agents/agent_yaml/map.go | 8 ++++---- .../pkg/agents/agent_yaml/map_voice_test.go | 14 +++++++------- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/doctor/checks_agent_status.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/doctor/checks_agent_status.go index 840792460b8..1ee42a84765 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/doctor/checks_agent_status.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/doctor/checks_agent_status.go @@ -704,7 +704,7 @@ func readAgentServices(prior []Result) []string { // filterHostedAgentServices removes prompt-voice services from the hosted-agent // status probe. The remote.agent-status check targets hosted agent versions -// (`/agents/{name}/versions/{version}`). Unified prompt-voice deploys record a +// (`/agents/{name}/versions/{version}`). Prompt-voice deploys record a // VERSION, but voice readiness follows its WebSocket endpoint rather than the // hosted-agent version lifecycle and should be covered by a voice-specific // doctor check in a future PR. diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/nextstep/state.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/nextstep/state.go index e06719e3330..613f57a306d 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/nextstep/state.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/nextstep/state.go @@ -37,7 +37,7 @@ const ( // agentEndpointVarFormat is the base endpoint env-var written for every // deployed agent. Voice agents (kind: prompt-voice) use it as the deploy - // completion marker. Unified voice deploys also set VERSION, while legacy + // completion marker. Prompt voice deploys also set VERSION, while legacy // voice environments may have only this endpoint marker. agentEndpointVarFormat = "AGENT_%s_ENDPOINT" @@ -827,7 +827,7 @@ func isDeployed( } // Voice deploys use the base ENDPOINT env var as the completion marker. - // Unified voice deploys write VERSION before ENDPOINT to keep ENDPOINT as the + // Prompt voice deploys write VERSION before ENDPOINT to keep ENDPOINT as the // final marker. // Require ENDPOINT for voice even when VERSION is present, otherwise a partial // env write could be reported as deployed before the callable endpoint was diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/nextstep/state_test.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/nextstep/state_test.go index 7f50fe6f190..2de778c98ac 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/nextstep/state_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/nextstep/state_test.go @@ -741,7 +741,7 @@ func TestServiceKey(t *testing.T) { // TestIsDeployed_VoiceEndpointFallback verifies that voice readiness is based // on the base endpoint marker. Legacy voice agents never set VERSION, while -// unified voice agents set VERSION before ENDPOINT; in both cases ENDPOINT is +// prompt voice agents set VERSION before ENDPOINT; in both cases ENDPOINT is // the deploy completion marker. func TestIsDeployed_VoiceEndpointFallback(t *testing.T) { t.Parallel() @@ -764,7 +764,7 @@ func TestIsDeployed_VoiceEndpointFallback(t *testing.T) { want: false, }, { - name: "version and endpoint set: deployed (unified voice agent)", + name: "version and endpoint set: deployed (prompt voice agent)", values: map[string]string{ "env1/AGENT_VOICE_SVC_VERSION": "1", "env1/AGENT_VOICE_SVC_ENDPOINT": "wss://x/agents/a/endpoint/protocols/voice?api-version=v1", @@ -774,7 +774,7 @@ func TestIsDeployed_VoiceEndpointFallback(t *testing.T) { }, { name: "no version but base endpoint set: deployed (voice agent)", - values: map[string]string{"env1/AGENT_VOICE_SVC_ENDPOINT": "https://x/voice_agents/a"}, + values: map[string]string{"env1/AGENT_VOICE_SVC_ENDPOINT": "wss://x/agents/a/endpoint/protocols/voice?api-version=v1"}, isVoice: true, want: true, }, diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go index 3225ef51fb2..1629e1d0264 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go @@ -583,7 +583,7 @@ func buildVoiceConfig(name string) *agent_api.VoiceConfig { return &agent_api.VoiceConfig{Type: "azure_standard", Name: trimmed} } -func flatVoiceType(voice *agent_api.VoiceConfig) string { +func voiceWireType(voice *agent_api.VoiceConfig) string { if voice == nil { return "" } @@ -593,7 +593,7 @@ func flatVoiceType(voice *agent_api.VoiceConfig) string { return voice.Type } -func flatVoiceLocale(voice *agent_api.VoiceConfig) string { +func voiceWireLocale(voice *agent_api.VoiceConfig) string { if voice == nil || voice.Name == "" || isOpenAIVoice(voice.Name) { return "" } @@ -664,8 +664,8 @@ func createVoiceAgentAPIRequest(voiceAgent VoiceAgent) (*agent_api.CreateAgentRe Output: &agent_api.VoiceOutputConfig{ Format: audioFormat, Voice: voiceConfig.Name, - VoiceType: flatVoiceType(voiceConfig), - VoiceLocale: flatVoiceLocale(voiceConfig), + VoiceType: voiceWireType(voiceConfig), + VoiceLocale: voiceWireLocale(voiceConfig), }, }, OutputModalities: []string{"audio"}, diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go index 5d51ac12822..0060e81ba60 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go @@ -173,11 +173,11 @@ func TestCreateVoiceAgentAPIRequest_Overrides(t *testing.T) { } } -func TestCreateVoiceAgentAPIRequest_UsesFlatOutputShape(t *testing.T) { +func TestCreateVoiceAgentAPIRequest_UsesServiceOutputShape(t *testing.T) { t.Parallel() voice := "alloy" agent := VoiceAgent{ - AgentDefinition: AgentDefinition{Kind: AgentKindPromptVoice, Name: "voice-flat"}, + AgentDefinition: AgentDefinition{Kind: AgentKindPromptVoice, Name: "voice-agent"}, Model: &Model{Id: "gpt-realtime"}, Voice: &voice, } @@ -198,11 +198,11 @@ func TestCreateVoiceAgentAPIRequest_UsesFlatOutputShape(t *testing.T) { } } -func TestCreateVoiceAgentAPIRequest_AzureVoiceLocale(t *testing.T) { +func TestCreateVoiceAgentAPIRequest_UsesAzureVoiceLocale(t *testing.T) { t.Parallel() voice := "en-US-Ava:DragonHDLatestNeural" agent := VoiceAgent{ - AgentDefinition: AgentDefinition{Kind: AgentKindPromptVoice, Name: "voice-flat"}, + AgentDefinition: AgentDefinition{Kind: AgentKindPromptVoice, Name: "voice-agent"}, Model: &Model{Id: "gpt-realtime"}, Voice: &voice, } @@ -223,11 +223,11 @@ func TestCreateVoiceAgentAPIRequest_AzureVoiceLocale(t *testing.T) { } } -func TestCreateVoiceAgentAPIRequest_MarshalWireShape(t *testing.T) { +func TestCreateVoiceAgentAPIRequest_MarshalServiceWireShape(t *testing.T) { t.Parallel() voice := "en-US-Ava:DragonHDLatestNeural" agent := VoiceAgent{ - AgentDefinition: AgentDefinition{Kind: AgentKindPromptVoice, Name: "voice-flat"}, + AgentDefinition: AgentDefinition{Kind: AgentKindPromptVoice, Name: "voice-agent"}, Model: &Model{Id: "gpt-realtime"}, Voice: &voice, } @@ -268,7 +268,7 @@ func TestCreateVoiceAgentAPIRequest_MarshalWireShape(t *testing.T) { t.Fatalf("audio.output.voice_locale = %#v, want en-US", got) } if _, exists := output["type"]; exists { - t.Fatalf("audio.output.type should not be present in flat wire shape: %#v", output) + t.Fatalf("audio.output.type should not be present in service wire shape: %#v", output) } } From cab0cb1829ab0803defe487d482c28b39d361d95 Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Tue, 25 Aug 2026 09:32:57 +0800 Subject: [PATCH 23/38] fix(agents): clear voice endpoint marker before writes --- .../internal/project/service_target_agent.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go index 4d2fb1f1ae4..97416a65d66 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go @@ -2214,10 +2214,18 @@ func (p *AgentServiceTargetProvider) deployVoiceAgent( baseEndpoint := buildVoiceWSProtocolURL(projectEndpoint, agentObject.Name) versionKey := fmt.Sprintf("AGENT_%s_VERSION", serviceKey) versionValue := agentObject.Versions.Latest.Version + endpointKey := fmt.Sprintf("AGENT_%s_ENDPOINT", serviceKey) + if _, setErr := p.azdClient.Environment().SetValue(ctx, &azdext.SetEnvRequest{ + EnvName: p.env.Name, + Key: endpointKey, + Value: "", + }); setErr != nil { + return nil, fmt.Errorf("clearing voice agent environment variable %s: %w", endpointKey, setErr) + } for _, envVar := range []struct{ key, value string }{ {fmt.Sprintf("AGENT_%s_NAME", serviceKey), agentObject.Name}, {versionKey, versionValue}, - {fmt.Sprintf("AGENT_%s_ENDPOINT", serviceKey), baseEndpoint}, + {endpointKey, baseEndpoint}, } { if _, setErr := p.azdClient.Environment().SetValue(ctx, &azdext.SetEnvRequest{ EnvName: p.env.Name, From 22e9f0d18c9e41169f8ffddbe25ef78e7f83f7aa Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Tue, 25 Aug 2026 09:56:01 +0800 Subject: [PATCH 24/38] fix(agents): complete voice API pivot --- .../azure.ai.agents/internal/exterrors/codes.go | 1 + .../internal/pkg/agents/agent_yaml/map.go | 15 +++++++-------- .../internal/project/service_target_agent.go | 3 ++- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/internal/exterrors/codes.go b/cli/azd/extensions/azure.ai.agents/internal/exterrors/codes.go index 477c65a5a67..a4837d7bea7 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/exterrors/codes.go +++ b/cli/azd/extensions/azure.ai.agents/internal/exterrors/codes.go @@ -194,6 +194,7 @@ const ( OpContainerPackage = "container_package" OpContainerPublish = "container_publish" OpCreateAgent = "create_agent" + OpGetAgent = "get_agent" OpUpdateAgent = "update_agent" OpGetActivityBot = "get_activity_bot" OpEnsureActivityBot = "ensure_activity_bot" diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go index 1629e1d0264..514d49f8cfb 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go @@ -549,11 +549,10 @@ var knownOpenAIVoices = map[string]struct{}{ } // azureNeuralVoicePattern matches the locale prefix that every Azure Neural -// voice name carries, e.g. "en-US-Ava:DragonHDLatestNeural" or -// "ja-JP-NanamiNeural". The service contract guarantees this -- -// shape for Azure voices, which is what distinguishes them from the flat -// lowercase OpenAI voice tokens. -var azureNeuralVoicePattern = regexp.MustCompile(`^[a-z]{2,3}-[A-Z]{2,3}-`) +// voice name carries, e.g. "en-US-Ava:DragonHDLatestNeural", +// "ja-JP-NanamiNeural", or Azure voices with script locales. The optional script tag +// keeps valid BCP-47 locales from being classified as OpenAI voices. +var azureNeuralVoicePattern = regexp.MustCompile(`^([a-z]{2,3}(?:-[A-Z][a-z]{3})?-[A-Z]{2,3})-`) // isOpenAIVoice reports whether a voice name denotes an OpenAI realtime voice // (e.g. "alloy") vs an Azure Neural voice (e.g. "en-US-Ava:DragonHDLatestNeural"). @@ -597,11 +596,11 @@ func voiceWireLocale(voice *agent_api.VoiceConfig) string { if voice == nil || voice.Name == "" || isOpenAIVoice(voice.Name) { return "" } - parts := strings.SplitN(voice.Name, "-", 3) - if len(parts) < 2 { + match := azureNeuralVoicePattern.FindStringSubmatch(voice.Name) + if len(match) < 2 { return "" } - return parts[0] + "-" + parts[1] + return match[1] } // CreateVoiceAgentAPIRequest builds a CreateAgentRequest for a declarative diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go index 97416a65d66..40cea42a0a6 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go @@ -2225,6 +2225,7 @@ func (p *AgentServiceTargetProvider) deployVoiceAgent( for _, envVar := range []struct{ key, value string }{ {fmt.Sprintf("AGENT_%s_NAME", serviceKey), agentObject.Name}, {versionKey, versionValue}, + {fmt.Sprintf("AGENT_%s_PROJECT_ENDPOINT", serviceKey), strings.TrimRight(projectEndpoint, "/")}, {endpointKey, baseEndpoint}, } { if _, setErr := p.azdClient.Environment().SetValue(ctx, &azdext.SetEnvRequest{ @@ -2276,7 +2277,7 @@ func (p *AgentServiceTargetProvider) deployVoiceAgentRemote( ) shouldUpdate, decisionErr := shouldUpdateVoiceAgent(remoteAgent, getErr) if decisionErr != nil { - return nil, exterrors.OpCreateAgent, decisionErr + return nil, exterrors.OpGetAgent, decisionErr } if shouldUpdate { progress("Updating voice agent using unified API") From 943bf68782bb0f5dab1ffc6b04ff7ac3f9999bd2 Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Tue, 25 Aug 2026 10:33:55 +0800 Subject: [PATCH 25/38] fix(agents): support numeric voice locales --- .../internal/pkg/agents/agent_yaml/map.go | 7 ++-- .../pkg/agents/agent_yaml/map_voice_test.go | 37 +++++++++++++++++++ 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go index 514d49f8cfb..cff9c5affe9 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go @@ -550,9 +550,10 @@ var knownOpenAIVoices = map[string]struct{}{ // azureNeuralVoicePattern matches the locale prefix that every Azure Neural // voice name carries, e.g. "en-US-Ava:DragonHDLatestNeural", -// "ja-JP-NanamiNeural", or Azure voices with script locales. The optional script tag -// keeps valid BCP-47 locales from being classified as OpenAI voices. -var azureNeuralVoicePattern = regexp.MustCompile(`^([a-z]{2,3}(?:-[A-Z][a-z]{3})?-[A-Z]{2,3})-`) +// "ja-JP-NanamiNeural", or Azure voices with script/numeric-region locales. +// The optional script tag and numeric region support keep valid BCP-47 locales +// from being classified as OpenAI voices. +var azureNeuralVoicePattern = regexp.MustCompile(`^([a-z]{2,3}(?:-[A-Z][a-z]{3})?-(?:[A-Z]{2,3}|[0-9]{3}))-`) // isOpenAIVoice reports whether a voice name denotes an OpenAI realtime voice // (e.g. "alloy") vs an Azure Neural voice (e.g. "en-US-Ava:DragonHDLatestNeural"). diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go index 0060e81ba60..4b603303bfb 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go @@ -223,6 +223,43 @@ func TestCreateVoiceAgentAPIRequest_UsesAzureVoiceLocale(t *testing.T) { } } +func TestCreateVoiceAgentAPIRequest_UsesAzureVoiceLocaleVariants(t *testing.T) { + t.Parallel() + tests := []struct { + name string + voice string + wantLocale string + }{ + {name: "script locale", voice: "az-Latn-AZ-BanuNeural", wantLocale: "az-Latn-AZ"}, + {name: "numeric region", voice: "es-419-AnaNeural", wantLocale: "es-419"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + agent := VoiceAgent{ + AgentDefinition: AgentDefinition{Kind: AgentKindPromptVoice, Name: "voice-agent"}, + Model: &Model{Id: "gpt-realtime"}, + Voice: &tt.voice, + } + + req, err := CreateVoiceAgentAPIRequest(agent) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + def := req.Definition.(agent_api.VoiceAgentDefinition) + if def.Audio.Output.Voice != tt.voice { + t.Errorf("Voice = %q, want %q", def.Audio.Output.Voice, tt.voice) + } + if def.Audio.Output.VoiceType != "azure-standard" { + t.Errorf("VoiceType = %q, want azure-standard", def.Audio.Output.VoiceType) + } + if def.Audio.Output.VoiceLocale != tt.wantLocale { + t.Errorf("VoiceLocale = %q, want %q", def.Audio.Output.VoiceLocale, tt.wantLocale) + } + }) + } +} + func TestCreateVoiceAgentAPIRequest_MarshalServiceWireShape(t *testing.T) { t.Parallel() voice := "en-US-Ava:DragonHDLatestNeural" From 4c2447bb3f3dda03b46966c2bad6dcfa7a5a87b9 Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Tue, 25 Aug 2026 10:43:29 +0800 Subject: [PATCH 26/38] test(agents): split voice endpoint fixture --- .../azure.ai.agents/internal/cmd/nextstep/state_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/nextstep/state_test.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/nextstep/state_test.go index 2de778c98ac..345998fbb48 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/nextstep/state_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/nextstep/state_test.go @@ -773,8 +773,10 @@ func TestIsDeployed_VoiceEndpointFallback(t *testing.T) { want: true, }, { - name: "no version but base endpoint set: deployed (voice agent)", - values: map[string]string{"env1/AGENT_VOICE_SVC_ENDPOINT": "wss://x/agents/a/endpoint/protocols/voice?api-version=v1"}, + name: "no version but base endpoint set: deployed (voice agent)", + values: map[string]string{ + "env1/AGENT_VOICE_SVC_ENDPOINT": "wss://x/agents/a/endpoint/protocols/voice?api-version=v1", + }, isVoice: true, want: true, }, From 94eb368851cc05d63d5a4d062b4629481483a61b Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Tue, 25 Aug 2026 12:09:01 +0800 Subject: [PATCH 27/38] feat(agents): support advanced voice settings --- .../extensions/azure.ai.agents/cspell.yaml | 3 + .../internal/pkg/agents/agent_api/models.go | 61 ++++-- .../internal/pkg/agents/agent_yaml/map.go | 180 ++++++++++++++++-- .../pkg/agents/agent_yaml/map_voice_test.go | 102 +++++++++- .../internal/pkg/agents/agent_yaml/parse.go | 89 +++++++++ .../internal/pkg/agents/agent_yaml/yaml.go | 89 +++++++++ .../internal/project/agent_definition.go | 67 +++++-- .../schemas/azure.ai.agent.json | 159 ++++++++++++++-- 8 files changed, 690 insertions(+), 60 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/cspell.yaml b/cli/azd/extensions/azure.ai.agents/cspell.yaml index 5d2d9e0950c..78be91d2fea 100644 --- a/cli/azd/extensions/azure.ai.agents/cspell.yaml +++ b/cli/azd/extensions/azure.ai.agents/cspell.yaml @@ -11,6 +11,9 @@ words: # Voice (prompt-voice) agents - BYOM - Nanami + - pcma + - pcmu + - webrtc # Azure region names - australiaeast - brazilsouth diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go index 86b452662d3..03750f4e94e 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go @@ -359,7 +359,7 @@ const ( // VoiceAudioFormat describes a PCM audio stream format (e.g. audio/pcm @ 24 kHz). type VoiceAudioFormat struct { Type string `json:"type"` - Rate int `json:"rate"` + Rate *int `json:"rate,omitempty"` } // VoiceTurnDetection configures server-side voice-activity detection so the @@ -369,26 +369,47 @@ type VoiceTurnDetection struct { Threshold *float64 `json:"threshold,omitempty"` PrefixPaddingMs *int `json:"prefix_padding_ms,omitempty"` SilenceDurationMs *int `json:"silence_duration_ms,omitempty"` + CreateResponse *bool `json:"create_response,omitempty"` + Eagerness *string `json:"eagerness,omitempty"` + SpeechDurationMs *int `json:"speech_duration_ms,omitempty"` + RemoveFillerWords *bool `json:"remove_filler_words,omitempty"` + InterruptResponse *bool `json:"interrupt_response,omitempty"` + Languages []string `json:"languages,omitempty"` + AutoTruncate *bool `json:"auto_truncate,omitempty"` } // VoiceTranscription enables user-speech transcription events on the input stream. type VoiceTranscription struct { - Model string `json:"model,omitempty"` + Model string `json:"model,omitempty"` + Language *string `json:"language,omitempty"` + Prompt *string `json:"prompt,omitempty"` +} + +// VoiceNoiseReduction configures input audio noise reduction. +type VoiceNoiseReduction struct { + Type string `json:"type"` } // VoiceInputConfig is the input (caller -> agent) audio configuration. type VoiceInputConfig struct { - Format *VoiceAudioFormat `json:"format,omitempty"` - TurnDetection *VoiceTurnDetection `json:"turn_detection,omitempty"` - Transcription *VoiceTranscription `json:"transcription,omitempty"` + Format *VoiceAudioFormat `json:"format,omitempty"` + NoiseReduction *VoiceNoiseReduction `json:"noise_reduction,omitempty"` + EchoCancellation map[string]any `json:"echo_cancellation,omitempty"` + TurnDetection *VoiceTurnDetection `json:"turn_detection,omitempty"` + Transcription *VoiceTranscription `json:"transcription,omitempty"` } // VoiceConfig selects the output voice. Type is "openai" for realtime voices // (single lowercase word, e.g. "alloy") or "azure_standard" for Azure Neural // voices (e.g. "en-US-Ava:DragonHDLatestNeural"). type VoiceConfig struct { - Type string `json:"type"` - Name string `json:"name"` + Type string `json:"type"` + Name string `json:"name"` + Style *string `json:"style,omitempty"` + Pitch *string `json:"pitch,omitempty"` + Rate *string `json:"rate,omitempty"` + Locale *string `json:"locale,omitempty"` + Volume *string `json:"volume,omitempty"` } // VoiceOutputConfig is the output (agent -> caller) audio configuration for the @@ -399,6 +420,11 @@ type VoiceOutputConfig struct { Voice string `json:"voice,omitempty"` VoiceType string `json:"voice_type,omitempty"` VoiceLocale string `json:"voice_locale,omitempty"` + Style *string `json:"style,omitempty"` + Pitch *string `json:"pitch,omitempty"` + Rate *string `json:"rate,omitempty"` + Volume *string `json:"volume,omitempty"` + Speed *float64 `json:"speed,omitempty"` } // VoiceAudioConfig bundles the input and output audio configuration. @@ -411,12 +437,21 @@ type VoiceAudioConfig struct { // prompt voice agent. Its Kind is always AgentKindVoice ("voice"). type VoiceAgentDefinition struct { AgentDefinition - ModelType VoiceModelType `json:"model_type"` - Model string `json:"model"` - Instructions string `json:"instructions,omitempty"` - Audio *VoiceAudioConfig `json:"audio,omitempty"` - OutputModalities []string `json:"output_modalities,omitempty"` - Store *bool `json:"store,omitempty"` + ModelType VoiceModelType `json:"model_type"` + Model string `json:"model"` + Instructions string `json:"instructions,omitempty"` + StructuredInputs map[string]any `json:"structured_inputs,omitempty"` + Audio *VoiceAudioConfig `json:"audio,omitempty"` + OutputModalities []string `json:"output_modalities,omitempty"` + Store *bool `json:"store,omitempty"` + Tools []map[string]any `json:"tools,omitempty"` + Avatar map[string]any `json:"avatar,omitempty"` + Greeting map[string]any `json:"greeting,omitempty"` + Handoff map[string]any `json:"handoff,omitempty"` + ToolChoice any `json:"tool_choice,omitempty"` + ParallelToolCalls *bool `json:"parallel_tool_calls,omitempty"` + MaxOutputTokens any `json:"max_output_tokens,omitempty"` + Include []string `json:"include,omitempty"` } // CreateAgentVersionRequest represents a request to create an agent version diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go index cff9c5affe9..cc5549a588c 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go @@ -604,6 +604,114 @@ func voiceWireLocale(voice *agent_api.VoiceConfig) string { return match[1] } +func defaultVoiceAudioFormat() *agent_api.VoiceAudioFormat { + rate := defaultVoiceAudioRate + return &agent_api.VoiceAudioFormat{Type: defaultVoiceAudioType, Rate: &rate} +} + +func mapVoiceAudioFormat(format *VoiceAudioFormat, fallback *agent_api.VoiceAudioFormat) *agent_api.VoiceAudioFormat { + out := &agent_api.VoiceAudioFormat{} + if fallback != nil { + *out = *fallback + } + if format != nil { + if strings.TrimSpace(format.Type) != "" { + out.Type = strings.TrimSpace(format.Type) + } + if format.Rate != nil { + out.Rate = format.Rate + } + } + return out +} + +func mapVoiceTurnDetection(turnDetection *VoiceTurnDetection) *agent_api.VoiceTurnDetection { + out := &agent_api.VoiceTurnDetection{Type: defaultVoiceTurnDetectionType} + if turnDetection == nil { + return out + } + if strings.TrimSpace(turnDetection.Type) != "" { + out.Type = strings.TrimSpace(turnDetection.Type) + } + out.Threshold = turnDetection.Threshold + out.PrefixPaddingMs = turnDetection.PrefixPaddingMs + out.SilenceDurationMs = turnDetection.SilenceDurationMs + out.CreateResponse = turnDetection.CreateResponse + out.Eagerness = turnDetection.Eagerness + out.SpeechDurationMs = turnDetection.SpeechDurationMs + out.RemoveFillerWords = turnDetection.RemoveFillerWords + out.InterruptResponse = turnDetection.InterruptResponse + out.Languages = turnDetection.Languages + out.AutoTruncate = turnDetection.AutoTruncate + return out +} + +func mapVoiceTranscription(transcription *VoiceTranscription) *agent_api.VoiceTranscription { + out := &agent_api.VoiceTranscription{Model: defaultVoiceInputTranscriptionModel} + if transcription == nil { + return out + } + if strings.TrimSpace(transcription.Model) != "" { + out.Model = strings.TrimSpace(transcription.Model) + } + out.Language = transcription.Language + out.Prompt = transcription.Prompt + return out +} + +func mapVoiceConfig(voice *VoiceConfig, fallbackName string) *agent_api.VoiceConfig { + if voice == nil { + return buildVoiceConfig(fallbackName) + } + name := strings.TrimSpace(voice.Name) + if name == "" { + name = fallbackName + } + voiceType := strings.TrimSpace(voice.Type) + if voiceType == "" { + out := buildVoiceConfig(name) + out.Style = voice.Style + out.Pitch = voice.Pitch + out.Rate = voice.Rate + out.Locale = voice.Locale + out.Volume = voice.Volume + return out + } + return &agent_api.VoiceConfig{ + Type: voiceType, + Name: name, + Style: voice.Style, + Pitch: voice.Pitch, + Rate: voice.Rate, + Locale: voice.Locale, + Volume: voice.Volume, + } +} + +func mapVoiceStructuredInputs(inputs map[string]any) map[string]any { + if len(inputs) == 0 { + return nil + } + out := make(map[string]any, len(inputs)) + for name, input := range inputs { + inputMap, ok := input.(map[string]any) + if !ok { + out[name] = input + continue + } + + mapped := maps.Clone(inputMap) + if value, ok := mapped["defaultValue"]; ok { + if _, hasSnakeCase := mapped["default_value"]; !hasSnakeCase { + mapped["default_value"] = value + } + delete(mapped, "defaultValue") + } + out[name] = mapped + } + return out +} + // CreateVoiceAgentAPIRequest builds a CreateAgentRequest for a declarative // voice agent. It translates the authoring kind "prompt-voice" into the // data-plane service kind "voice" and defaults the audio pipeline. @@ -640,36 +748,76 @@ func createVoiceAgentAPIRequest(voiceAgent VoiceAgent) (*agent_api.CreateAgentRe voiceName = *voiceAgent.Voice } - audioFormat := &agent_api.VoiceAudioFormat{ - Type: defaultVoiceAudioType, - Rate: defaultVoiceAudioRate, + inputFormat := defaultVoiceAudioFormat() + outputFormat := defaultVoiceAudioFormat() + turnDetection := mapVoiceTurnDetection(nil) + transcription := mapVoiceTranscription(nil) + var noiseReduction *agent_api.VoiceNoiseReduction + var echoCancellation map[string]any + outputVoice := buildVoiceConfig(voiceName) + var outputSpeed *float64 + if voiceAgent.Audio != nil { + if voiceAgent.Audio.Input != nil { + inputFormat = mapVoiceAudioFormat(voiceAgent.Audio.Input.Format, inputFormat) + if voiceAgent.Audio.Input.NoiseReduction != nil { + noiseReduction = &agent_api.VoiceNoiseReduction{Type: strings.TrimSpace(voiceAgent.Audio.Input.NoiseReduction.Type)} + } + echoCancellation = voiceAgent.Audio.Input.EchoCancellation + turnDetection = mapVoiceTurnDetection(voiceAgent.Audio.Input.TurnDetection) + transcription = mapVoiceTranscription(voiceAgent.Audio.Input.Transcription) + } + if voiceAgent.Audio.Output != nil { + outputFormat = mapVoiceAudioFormat(voiceAgent.Audio.Output.Format, outputFormat) + outputVoice = mapVoiceConfig(voiceAgent.Audio.Output.Voice, voiceName) + outputSpeed = voiceAgent.Audio.Output.Speed + } + } + + outputModalities := []string{"audio"} + if len(voiceAgent.OutputModalities) > 0 { + outputModalities = voiceAgent.OutputModalities } input := &agent_api.VoiceInputConfig{ - Format: audioFormat, - TurnDetection: &agent_api.VoiceTurnDetection{Type: defaultVoiceTurnDetectionType}, - Transcription: &agent_api.VoiceTranscription{Model: defaultVoiceInputTranscriptionModel}, + Format: inputFormat, + NoiseReduction: noiseReduction, + EchoCancellation: echoCancellation, + TurnDetection: turnDetection, + Transcription: transcription, } - voiceConfig := buildVoiceConfig(voiceName) voiceDef := agent_api.VoiceAgentDefinition{ AgentDefinition: agent_api.AgentDefinition{ // Translate authoring kind prompt-voice -> service kind voice. Kind: agent_api.AgentKindVoice, }, - ModelType: modelType, - Model: modelID, - Instructions: instructions, + ModelType: modelType, + Model: modelID, + Instructions: instructions, + StructuredInputs: mapVoiceStructuredInputs(voiceAgent.StructuredInputs), Audio: &agent_api.VoiceAudioConfig{ Input: input, Output: &agent_api.VoiceOutputConfig{ - Format: audioFormat, - Voice: voiceConfig.Name, - VoiceType: voiceWireType(voiceConfig), - VoiceLocale: voiceWireLocale(voiceConfig), + Format: outputFormat, + Voice: outputVoice.Name, + VoiceType: voiceWireType(outputVoice), + VoiceLocale: voiceWireLocale(outputVoice), + Style: outputVoice.Style, + Pitch: outputVoice.Pitch, + Rate: outputVoice.Rate, + Volume: outputVoice.Volume, + Speed: outputSpeed, }, }, - OutputModalities: []string{"audio"}, - Store: voiceAgent.Store, + OutputModalities: outputModalities, + Store: voiceAgent.Store, + Tools: voiceAgent.Tools, + Avatar: voiceAgent.Avatar, + Greeting: voiceAgent.Greeting, + Handoff: voiceAgent.Handoff, + ToolChoice: voiceAgent.ToolChoice, + ParallelToolCalls: voiceAgent.ParallelToolCalls, + MaxOutputTokens: voiceAgent.MaxOutputTokens, + Include: voiceAgent.Include, } return createAgentAPIRequest(voiceAgent.AgentDefinition, voiceDef, nil, nil) diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go index 4b603303bfb..b507e68b377 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go @@ -10,6 +10,8 @@ import ( "azureaiagent/internal/pkg/agents/agent_api" ) +func ptr[T any](v T) *T { return &v } + // --------------------------------------------------------------------------- // isOpenAIVoice / buildVoiceConfig // --------------------------------------------------------------------------- @@ -118,7 +120,8 @@ func TestCreateVoiceAgentAPIRequest_Defaults(t *testing.T) { t.Fatalf("Audio pipeline not populated: %+v", def.Audio) } in := def.Audio.Input - if in.Format == nil || in.Format.Type != defaultVoiceAudioType || in.Format.Rate != defaultVoiceAudioRate { + if in.Format == nil || in.Format.Type != defaultVoiceAudioType || in.Format.Rate == nil || + *in.Format.Rate != defaultVoiceAudioRate { t.Errorf("input format = %+v", in.Format) } if in.TurnDetection == nil || in.TurnDetection.Type != defaultVoiceTurnDetectionType { @@ -128,7 +131,8 @@ func TestCreateVoiceAgentAPIRequest_Defaults(t *testing.T) { t.Errorf("transcription = %+v", in.Transcription) } out := def.Audio.Output - if out.Format == nil || out.Format.Type != defaultVoiceAudioType || out.Format.Rate != defaultVoiceAudioRate { + if out.Format == nil || out.Format.Type != defaultVoiceAudioType || out.Format.Rate == nil || + *out.Format.Rate != defaultVoiceAudioRate { t.Errorf("output format = %+v", out.Format) } // Default voice is the DragonHD Azure Neural voice in the flat unified shape. @@ -309,6 +313,100 @@ func TestCreateVoiceAgentAPIRequest_MarshalServiceWireShape(t *testing.T) { } } +func TestCreateVoiceAgentAPIRequest_AdvancedSettingsWireShape(t *testing.T) { + t.Parallel() + inRate := 16000 + outRate := 24000 + threshold := 0.6 + speechDurationMs := 120 + createResponse := true + removeFillerWords := true + interruptResponse := true + autoTruncate := true + speed := 1.1 + parallelToolCalls := true + style := "cheerful" + pitch := "+0Hz" + rate := "+0%" + volume := "+0%" + agent := VoiceAgent{ + AgentDefinition: AgentDefinition{Kind: AgentKindPromptVoice, Name: "voice-advanced"}, + Model: &Model{Id: "gpt-realtime"}, + Instructions: ptr("You are {{persona}}, a concise voice assistant."), + StructuredInputs: map[string]any{ + "persona": map[string]any{"description": "Assistant persona", "defaultValue": "Ada"}, + }, + Audio: &VoiceAudio{ + Input: &VoiceAudioInput{ + Format: &VoiceAudioFormat{Type: "audio/pcmu", Rate: &inRate}, + NoiseReduction: &VoiceNoiseReduction{Type: "near_field"}, + EchoCancellation: map[string]any{"type": "server_echo_cancellation", "channels": 1}, + TurnDetection: &VoiceTurnDetection{ + Type: "azure_semantic_vad", + Threshold: &threshold, + SpeechDurationMs: &speechDurationMs, + CreateResponse: &createResponse, + RemoveFillerWords: &removeFillerWords, + InterruptResponse: &interruptResponse, + Languages: []string{"en-US"}, + AutoTruncate: &autoTruncate, + }, + Transcription: &VoiceTranscription{Model: "whisper-1", Language: ptr("en-US"), Prompt: ptr("Contoso terms")}, + }, + Output: &VoiceAudioOutput{ + Format: &VoiceAudioFormat{Type: "audio/pcm", Rate: &outRate}, + Voice: &VoiceConfig{ + Type: "azure_standard", Name: "en-US-AvaNeural", Style: &style, + Pitch: &pitch, Rate: &rate, Locale: ptr("en-US"), Volume: &volume, + }, + Speed: &speed, + }, + }, + OutputModalities: []string{"audio", "text"}, + Tools: []map[string]any{{"type": "system", "name": "end_conversation"}}, + Avatar: map[string]any{"type": "video_avatar", "character": "lisa", "output_protocol": "webrtc"}, + Greeting: map[string]any{"type": "template", "text": "Hello {{persona}}"}, + ToolChoice: "auto", + ParallelToolCalls: ¶llelToolCalls, + MaxOutputTokens: "inf", + Include: []string{"item.input_audio_transcription.phrases"}, + } + + req, err := CreateVoiceAgentAPIRequest(agent) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + payload, err := json.Marshal(req) + if err != nil { + t.Fatalf("marshal request: %v", err) + } + + var wire map[string]any + if err := json.Unmarshal(payload, &wire); err != nil { + t.Fatalf("unmarshal payload: %v", err) + } + def := wire["definition"].(map[string]any) + input := def["audio"].(map[string]any)["input"].(map[string]any) + output := def["audio"].(map[string]any)["output"].(map[string]any) + structured := def["structured_inputs"].(map[string]any)["persona"].(map[string]any) + + if structured["default_value"] != "Ada" || structured["defaultValue"] != nil { + t.Fatalf("structured input default was not mapped to wire shape: %#v", structured) + } + if output["voice"] != "en-US-AvaNeural" || output["voice_type"] != "azure-standard" || output["style"] != style { + t.Fatalf("output voice flat shape not mapped: %#v", output) + } + if input["echo_cancellation"].(map[string]any)["type"] != "server_echo_cancellation" { + t.Fatalf("echo cancellation not mapped: %#v", input["echo_cancellation"]) + } + if def["tool_choice"] != "auto" || def["max_output_tokens"] != "inf" { + t.Fatalf("response options not mapped: %#v", def) + } + if len(def["tools"].([]any)) != 1 || def["avatar"].(map[string]any)["character"] != "lisa" { + t.Fatalf("tools/avatar not mapped: %#v", def) + } +} + // TestCreateVoiceAgentAPIRequest_ExplicitManaged verifies that explicitly // setting model_type: managed is accepted (idempotent with the default). func TestCreateVoiceAgentAPIRequest_ExplicitManaged(t *testing.T) { diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go index 996eae8a561..a90fbe6046d 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go @@ -460,6 +460,7 @@ func ValidateAgentDefinition(templateBytes []byte) error { "template.model_type '%s' is not supported; use '%s' or '%s'", agent.ModelType, VoiceModelTypeManaged, VoiceModelTypeSelfDeployed)) } + errors = append(errors, validateVoiceAgentAdvancedConfig(agent)...) } else { errors = append(errors, fmt.Sprintf("failed to unmarshal to VoiceAgent: %v", err)) } @@ -479,6 +480,94 @@ func ValidateAgentDefinition(templateBytes []byte) error { return nil } +func validateVoiceAgentAdvancedConfig(agent VoiceAgent) []string { + var errors []string + for i, modality := range agent.OutputModalities { + if strings.TrimSpace(modality) == "" { + errors = append(errors, fmt.Sprintf("template.output_modalities[%d] must not be blank", i)) + } + } + + if agent.Audio == nil { + return append(errors, validateVoiceIncludeTranscriptionCompatibility(agent, "")...) + } + transcriptionModel := "" + if agent.Audio.Input != nil { + errors = append(errors, validateVoiceAudioFormat("template.audio.input.format", agent.Audio.Input.Format)...) + if nr := agent.Audio.Input.NoiseReduction; nr != nil && strings.TrimSpace(nr.Type) == "" { + errors = append(errors, "template.audio.input.noise_reduction.type must not be blank") + } + if td := agent.Audio.Input.TurnDetection; td != nil { + if strings.TrimSpace(td.Type) == "" { + errors = append(errors, "template.audio.input.turn_detection.type must not be blank") + } + if td.Threshold != nil && (*td.Threshold < 0 || *td.Threshold > 1) { + errors = append(errors, "template.audio.input.turn_detection.threshold must be between 0 and 1") + } + if td.PrefixPaddingMs != nil && *td.PrefixPaddingMs < 0 { + errors = append(errors, "template.audio.input.turn_detection.prefix_padding_ms must be >= 0") + } + if td.SilenceDurationMs != nil && *td.SilenceDurationMs < 0 { + errors = append(errors, "template.audio.input.turn_detection.silence_duration_ms must be >= 0") + } + if td.SpeechDurationMs != nil && *td.SpeechDurationMs < 0 { + errors = append(errors, "template.audio.input.turn_detection.speech_duration_ms must be >= 0") + } + } + if agent.Audio.Input.Transcription != nil { + transcriptionModel = agent.Audio.Input.Transcription.Model + } + } + if agent.Audio.Output != nil { + errors = append(errors, validateVoiceAudioFormat("template.audio.output.format", agent.Audio.Output.Format)...) + if voice := agent.Audio.Output.Voice; voice != nil { + if strings.TrimSpace(voice.Type) == "" { + errors = append(errors, "template.audio.output.voice.type must not be blank") + } + if strings.TrimSpace(voice.Name) == "" { + errors = append(errors, "template.audio.output.voice.name must not be blank") + } + } + if speed := agent.Audio.Output.Speed; speed != nil && (*speed < 0.25 || *speed > 1.5) { + errors = append(errors, "template.audio.output.speed must be between 0.25 and 1.5") + } + } + return append(errors, validateVoiceIncludeTranscriptionCompatibility(agent, transcriptionModel)...) +} + +func validateVoiceIncludeTranscriptionCompatibility(agent VoiceAgent, transcriptionModel string) []string { + if !slices.Contains(agent.Include, "item.input_audio_transcription.phrases") { + return nil + } + model := strings.TrimSpace(transcriptionModel) + if model == "" { + model = defaultVoiceInputTranscriptionModel + } + if model == "azure-speech" || model == "azure-fast-transcription" { + return nil + } + return []string{ + "template.include item.input_audio_transcription.phrases requires template.audio.input.transcription.model to be azure-speech or azure-fast-transcription", + } +} + +func validateVoiceAudioFormat(path string, format *VoiceAudioFormat) []string { + if format == nil { + return nil + } + var errors []string + formatType := strings.TrimSpace(format.Type) + if formatType == "" { + errors = append(errors, path+".type must not be blank") + } else if formatType != "audio/pcm" && formatType != "audio/pcmu" && formatType != "audio/pcma" { + errors = append(errors, path+".type must be 'audio/pcm', 'audio/pcmu', or 'audio/pcma'") + } + if format.Rate != nil && *format.Rate <= 0 { + errors = append(errors, path+".rate must be greater than 0") + } + return errors +} + // Validate that the agent name matches the expected deployable format func ValidateAgentName(name string) error { if name == "" { diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/yaml.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/yaml.go index 46fffca9e6a..ffa35d057db 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/yaml.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/yaml.go @@ -214,9 +214,98 @@ type VoiceAgent struct { // Voice is the output voice name (e.g. "en-US-Ava:DragonHDLatestNeural" for // an Azure Neural voice, or "alloy" for an OpenAI realtime voice). Voice *string `json:"voice,omitempty" yaml:"voice,omitempty"` + // StructuredInputs declares template inputs used by voice instructions and greeting. + StructuredInputs map[string]any `json:"structuredInputs,omitempty" yaml:"structured_inputs,omitempty"` + // Audio customizes the input and output voice pipeline. Missing fields keep azd defaults. + Audio *VoiceAudio `json:"audio,omitempty" yaml:"audio,omitempty"` + // OutputModalities declares response modalities such as audio, text, animation, or avatar. + OutputModalities []string `json:"outputModalities,omitempty" yaml:"output_modalities,omitempty"` // Store toggles server-side logging (transcript + per-turn audio). Optional; // the service defaults to false when omitted. Store *bool `json:"store,omitempty" yaml:"store,omitempty"` + // Tools are passed through to the prompt voice service. Supported direct tool + // types include function, mcp, system, and toolbox. + Tools []map[string]any `json:"tools,omitempty" yaml:"tools,omitempty"` + // Avatar customizes voice avatar output for services that support it. + Avatar map[string]any `json:"avatar,omitempty" yaml:"avatar,omitempty"` + // Greeting configures initial greeting behavior for services that support it. + Greeting map[string]any `json:"greeting,omitempty" yaml:"greeting,omitempty"` + // Handoff configures voice handoff behavior for services that support it. + Handoff map[string]any `json:"handoff,omitempty" yaml:"handoff,omitempty"` + // ToolChoice configures service tool choice behavior, such as auto/none/required. + ToolChoice any `json:"toolChoice,omitempty" yaml:"tool_choice,omitempty"` + // ParallelToolCalls toggles parallel tool calls. + ParallelToolCalls *bool `json:"parallelToolCalls,omitempty" yaml:"parallel_tool_calls,omitempty"` + // MaxOutputTokens limits response output tokens. Use an integer or service-supported string such as "inf". + MaxOutputTokens any `json:"maxOutputTokens,omitempty" yaml:"max_output_tokens,omitempty"` + // Include requests additional service response fields. + Include []string `json:"include,omitempty" yaml:"include,omitempty"` +} + +// VoiceAudio bundles optional prompt voice input/output audio overrides. +type VoiceAudio struct { + Input *VoiceAudioInput `json:"input,omitempty" yaml:"input,omitempty"` + Output *VoiceAudioOutput `json:"output,omitempty" yaml:"output,omitempty"` +} + +// VoiceAudioInput customizes caller-to-agent audio. +type VoiceAudioInput struct { + Format *VoiceAudioFormat `json:"format,omitempty" yaml:"format,omitempty"` + NoiseReduction *VoiceNoiseReduction `json:"noiseReduction,omitempty" yaml:"noise_reduction,omitempty"` + EchoCancellation map[string]any `json:"echoCancellation,omitempty" yaml:"echo_cancellation,omitempty"` + TurnDetection *VoiceTurnDetection `json:"turnDetection,omitempty" yaml:"turn_detection,omitempty"` + Transcription *VoiceTranscription `json:"transcription,omitempty" yaml:"transcription,omitempty"` +} + +// VoiceAudioOutput customizes agent-to-caller audio. +type VoiceAudioOutput struct { + Format *VoiceAudioFormat `json:"format,omitempty" yaml:"format,omitempty"` + Voice *VoiceConfig `json:"voice,omitempty" yaml:"voice,omitempty"` + Speed *float64 `json:"speed,omitempty" yaml:"speed,omitempty"` +} + +// VoiceAudioFormat describes an audio stream format. +type VoiceAudioFormat struct { + Type string `json:"type" yaml:"type"` + Rate *int `json:"rate,omitempty" yaml:"rate,omitempty"` +} + +// VoiceNoiseReduction configures input audio noise reduction. +type VoiceNoiseReduction struct { + Type string `json:"type" yaml:"type"` +} + +// VoiceTurnDetection configures server-side turn detection. +type VoiceTurnDetection struct { + Type string `json:"type" yaml:"type"` + Threshold *float64 `json:"threshold,omitempty" yaml:"threshold,omitempty"` + PrefixPaddingMs *int `json:"prefixPaddingMs,omitempty" yaml:"prefix_padding_ms,omitempty"` + SilenceDurationMs *int `json:"silenceDurationMs,omitempty" yaml:"silence_duration_ms,omitempty"` + CreateResponse *bool `json:"createResponse,omitempty" yaml:"create_response,omitempty"` + Eagerness *string `json:"eagerness,omitempty" yaml:"eagerness,omitempty"` + SpeechDurationMs *int `json:"speechDurationMs,omitempty" yaml:"speech_duration_ms,omitempty"` + RemoveFillerWords *bool `json:"removeFillerWords,omitempty" yaml:"remove_filler_words,omitempty"` + InterruptResponse *bool `json:"interruptResponse,omitempty" yaml:"interrupt_response,omitempty"` + Languages []string `json:"languages,omitempty" yaml:"languages,omitempty"` + AutoTruncate *bool `json:"autoTruncate,omitempty" yaml:"auto_truncate,omitempty"` +} + +// VoiceTranscription configures input transcription. +type VoiceTranscription struct { + Model string `json:"model,omitempty" yaml:"model,omitempty"` + Language *string `json:"language,omitempty" yaml:"language,omitempty"` + Prompt *string `json:"prompt,omitempty" yaml:"prompt,omitempty"` +} + +// VoiceConfig selects the output voice. +type VoiceConfig struct { + Type string `json:"type" yaml:"type"` + Name string `json:"name" yaml:"name"` + Style *string `json:"style,omitempty" yaml:"style,omitempty"` + Pitch *string `json:"pitch,omitempty" yaml:"pitch,omitempty"` + Rate *string `json:"rate,omitempty" yaml:"rate,omitempty"` + Locale *string `json:"locale,omitempty" yaml:"locale,omitempty"` + Volume *string `json:"volume,omitempty" yaml:"volume,omitempty"` } // ContainerResources represents the resource allocation for a containerized agent. diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/agent_definition.go b/cli/azd/extensions/azure.ai.agents/internal/project/agent_definition.go index 26b5a06f874..1625d9a7637 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/agent_definition.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/agent_definition.go @@ -138,35 +138,68 @@ type AgentDefinitionInline struct { // Voice-agent fields (kind: prompt-voice). All omitempty so container/ // workflow entries are byte-for-byte unchanged. - ModelType agent_yaml.VoiceModelType `json:"modelType,omitempty"` - Model *agent_yaml.Model `json:"model,omitempty"` - Instructions *string `json:"instructions,omitempty"` - Voice *string `json:"voice,omitempty"` - Store *bool `json:"store,omitempty"` + ModelType agent_yaml.VoiceModelType `json:"modelType,omitempty"` + Model *agent_yaml.Model `json:"model,omitempty"` + Instructions *string `json:"instructions,omitempty"` + Voice *string `json:"voice,omitempty"` + StructuredInputs map[string]any `json:"structuredInputs,omitempty"` + Audio *agent_yaml.VoiceAudio `json:"audio,omitempty"` + OutputModalities []string `json:"outputModalities,omitempty"` + Store *bool `json:"store,omitempty"` + Tools []map[string]any `json:"tools,omitempty"` + Avatar map[string]any `json:"avatar,omitempty"` + Greeting map[string]any `json:"greeting,omitempty"` + Handoff map[string]any `json:"handoff,omitempty"` + ToolChoice any `json:"toolChoice,omitempty"` + ParallelToolCalls *bool `json:"parallelToolCalls,omitempty"` + MaxOutputTokens any `json:"maxOutputTokens,omitempty"` + Include []string `json:"include,omitempty"` } // voiceAgentDefinitionToInline projects a VoiceAgent into the inline definition // written to azure.yaml. Voice agents carry no container/image/code config. func voiceAgentDefinitionToInline(va agent_yaml.VoiceAgent) AgentDefinitionInline { return AgentDefinitionInline{ - AgentDefinition: va.AgentDefinition, - ModelType: va.ModelType, - Model: va.Model, - Instructions: va.Instructions, - Voice: va.Voice, - Store: va.Store, + AgentDefinition: va.AgentDefinition, + ModelType: va.ModelType, + Model: va.Model, + Instructions: va.Instructions, + Voice: va.Voice, + StructuredInputs: va.StructuredInputs, + Audio: va.Audio, + OutputModalities: va.OutputModalities, + Store: va.Store, + Tools: va.Tools, + Avatar: va.Avatar, + Greeting: va.Greeting, + Handoff: va.Handoff, + ToolChoice: va.ToolChoice, + ParallelToolCalls: va.ParallelToolCalls, + MaxOutputTokens: va.MaxOutputTokens, + Include: va.Include, } } // toVoiceAgent rebuilds an agent_yaml.VoiceAgent from the inline definition. func (d AgentDefinitionInline) toVoiceAgent() agent_yaml.VoiceAgent { return agent_yaml.VoiceAgent{ - AgentDefinition: d.AgentDefinition, - ModelType: d.ModelType, - Model: d.Model, - Instructions: d.Instructions, - Voice: d.Voice, - Store: d.Store, + AgentDefinition: d.AgentDefinition, + ModelType: d.ModelType, + Model: d.Model, + Instructions: d.Instructions, + Voice: d.Voice, + StructuredInputs: d.StructuredInputs, + Audio: d.Audio, + OutputModalities: d.OutputModalities, + Store: d.Store, + Tools: d.Tools, + Avatar: d.Avatar, + Greeting: d.Greeting, + Handoff: d.Handoff, + ToolChoice: d.ToolChoice, + ParallelToolCalls: d.ParallelToolCalls, + MaxOutputTokens: d.MaxOutputTokens, + Include: d.Include, } } diff --git a/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json b/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json index 55a8ea2c55d..6eca9a0ef5a 100644 --- a/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json +++ b/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json @@ -64,14 +64,62 @@ "type": "string", "description": "Voice agent (kind: prompt-voice) system prompt for the assistant." }, - "voice": { - "type": "string", - "description": "Voice agent (kind: prompt-voice) output voice name (e.g. 'en-US-Ava:DragonHDLatestNeural' for an Azure Neural voice, or 'alloy' for an OpenAI realtime voice)." - }, - "store": { - "type": "boolean", - "description": "Voice agent (kind: prompt-voice) server-side logging toggle (transcript + per-turn audio). Defaults to false when omitted." - }, + "voice": { + "type": "string", + "description": "Voice agent (kind: prompt-voice) output voice name (e.g. 'en-US-Ava:DragonHDLatestNeural' for an Azure Neural voice, or 'alloy' for an OpenAI realtime voice)." + }, + "structuredInputs": { + "type": "object", + "description": "Voice agent (kind: prompt-voice) structured prompt inputs. Use description, defaultValue, schema, and required; azd maps defaultValue to the service wire field default_value.", + "additionalProperties": true + }, + "audio": { + "$ref": "#/definitions/VoiceAudio" + }, + "outputModalities": { + "type": "array", + "description": "Voice agent (kind: prompt-voice) output modalities. Well-known values are audio, text, animation, and avatar.", + "items": { "type": "string", "minLength": 1 } + }, + "store": { + "type": "boolean", + "description": "Voice agent (kind: prompt-voice) server-side logging toggle (transcript + per-turn audio). Defaults to false when omitted." + }, + "tools": { + "type": "array", + "description": "Voice agent (kind: prompt-voice) tools. Direct tool types include function, mcp, system, and toolbox.", + "items": { "type": "object", "additionalProperties": true } + }, + "avatar": { + "type": "object", + "description": "Voice agent (kind: prompt-voice) avatar configuration.", + "additionalProperties": true + }, + "greeting": { + "type": "object", + "description": "Voice agent (kind: prompt-voice) greeting configuration, such as template or llm_generated.", + "additionalProperties": true + }, + "handoff": { + "type": "object", + "description": "Voice agent (kind: prompt-voice) handoff configuration.", + "additionalProperties": true + }, + "toolChoice": { + "description": "Voice agent (kind: prompt-voice) tool choice behavior, such as none, auto, required, or a tool choice object." + }, + "parallelToolCalls": { + "type": "boolean", + "description": "Voice agent (kind: prompt-voice) parallel tool call toggle." + }, + "maxOutputTokens": { + "description": "Voice agent (kind: prompt-voice) maximum output tokens. Use an integer or a service-supported string such as inf." + }, + "include": { + "type": "array", + "description": "Voice agent (kind: prompt-voice) extra service response fields to include.", + "items": { "type": "string" } + }, "name": { "type": "string", "description": "The agent name." @@ -152,7 +200,7 @@ "required": ["protocol"], "additionalProperties": false }, - "CodeConfiguration": { + "CodeConfiguration": { "type": "object", "description": "Code deploy configuration. When present, the agent is deployed from source (ZIP) instead of a container image.", "properties": { @@ -161,9 +209,96 @@ "dependencyResolution": { "type": "string", "description": "Dependency resolution mode (e.g., 'bundled', 'remote_build')." } }, "required": ["runtime", "entryPoint"], - "additionalProperties": false - }, - "SessionConfiguration": { + "additionalProperties": false + }, + "VoiceAudio": { + "type": "object", + "description": "Prompt voice input and output audio configuration. Requires AZURE_VOICE_AGENT_API=unified-flat for deployment.", + "properties": { + "input": { "$ref": "#/definitions/VoiceAudioInput" }, + "output": { "$ref": "#/definitions/VoiceAudioOutput" } + }, + "additionalProperties": false + }, + "VoiceAudioInput": { + "type": "object", + "properties": { + "format": { "$ref": "#/definitions/VoiceAudioFormat" }, + "noiseReduction": { "$ref": "#/definitions/VoiceNoiseReduction" }, + "echoCancellation": { "type": "object", "additionalProperties": true }, + "turnDetection": { "$ref": "#/definitions/VoiceTurnDetection" }, + "transcription": { "$ref": "#/definitions/VoiceTranscription" } + }, + "additionalProperties": false + }, + "VoiceAudioOutput": { + "type": "object", + "properties": { + "format": { "$ref": "#/definitions/VoiceAudioFormat" }, + "voice": { "$ref": "#/definitions/VoiceConfig" }, + "speed": { "type": "number", "minimum": 0.25, "maximum": 1.5 } + }, + "additionalProperties": false + }, + "VoiceAudioFormat": { + "type": "object", + "properties": { + "type": { "type": "string", "enum": ["audio/pcm", "audio/pcmu", "audio/pcma"] }, + "rate": { "type": "integer", "minimum": 1 } + }, + "required": ["type"], + "additionalProperties": false + }, + "VoiceNoiseReduction": { + "type": "object", + "properties": { + "type": { "type": "string", "description": "Well-known values include near_field, far_field, and azure_deep_noise_suppression." } + }, + "required": ["type"], + "additionalProperties": false + }, + "VoiceTurnDetection": { + "type": "object", + "properties": { + "type": { "type": "string", "description": "Well-known values include server_vad, semantic_vad, and azure_semantic_vad." }, + "threshold": { "type": "number", "minimum": 0, "maximum": 1 }, + "prefixPaddingMs": { "type": "integer", "minimum": 0 }, + "silenceDurationMs": { "type": "integer", "minimum": 0 }, + "createResponse": { "type": "boolean" }, + "eagerness": { "type": "string" }, + "speechDurationMs": { "type": "integer", "minimum": 0 }, + "removeFillerWords": { "type": "boolean" }, + "interruptResponse": { "type": "boolean" }, + "languages": { "type": "array", "items": { "type": "string" } }, + "autoTruncate": { "type": "boolean" } + }, + "required": ["type"], + "additionalProperties": false + }, + "VoiceTranscription": { + "type": "object", + "properties": { + "model": { "type": "string" }, + "language": { "type": "string" }, + "prompt": { "type": "string" } + }, + "additionalProperties": false + }, + "VoiceConfig": { + "type": "object", + "properties": { + "type": { "type": "string", "description": "Voice provider type, such as openai, azure_standard, or azure-standard." }, + "name": { "type": "string" }, + "style": { "type": "string" }, + "pitch": { "type": "string" }, + "rate": { "type": "string" }, + "locale": { "type": "string" }, + "volume": { "type": "string" } + }, + "required": ["type", "name"], + "additionalProperties": false + }, + "SessionConfiguration": { "type": "object", "description": "Optional hosted-agent session runtime settings. When omitted, the service applies its defaults (idle timeout 900 seconds).", "properties": { From aba916c6adcf6a58e8ff4d25857bf3d182898872 Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Tue, 25 Aug 2026 12:23:12 +0800 Subject: [PATCH 28/38] test(agents): remove stale pointer helper --- .../internal/pkg/agents/agent_yaml/map_voice_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go index cc79acab936..a7caf58bc41 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go @@ -10,9 +10,6 @@ import ( "azureaiagent/internal/pkg/agents/agent_api" ) -//go:fix inline -func ptr[T any](v T) *T { return new(v) } - // --------------------------------------------------------------------------- // isOpenAIVoice / buildVoiceConfig // --------------------------------------------------------------------------- From 4d9da7f0e99ff530bcc52a373e34323d9788209e Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Tue, 25 Aug 2026 12:09:01 +0800 Subject: [PATCH 29/38] feat(agents): support advanced voice settings --- .../extensions/azure.ai.agents/cspell.yaml | 3 + .../internal/pkg/agents/agent_api/models.go | 61 ++++-- .../internal/pkg/agents/agent_yaml/map.go | 180 ++++++++++++++++-- .../pkg/agents/agent_yaml/map_voice_test.go | 102 +++++++++- .../internal/pkg/agents/agent_yaml/parse.go | 89 +++++++++ .../internal/pkg/agents/agent_yaml/yaml.go | 89 +++++++++ .../internal/project/agent_definition.go | 67 +++++-- .../schemas/azure.ai.agent.json | 135 +++++++++++++ 8 files changed, 678 insertions(+), 48 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/cspell.yaml b/cli/azd/extensions/azure.ai.agents/cspell.yaml index 5d2d9e0950c..78be91d2fea 100644 --- a/cli/azd/extensions/azure.ai.agents/cspell.yaml +++ b/cli/azd/extensions/azure.ai.agents/cspell.yaml @@ -11,6 +11,9 @@ words: # Voice (prompt-voice) agents - BYOM - Nanami + - pcma + - pcmu + - webrtc # Azure region names - australiaeast - brazilsouth diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go index 86b452662d3..03750f4e94e 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go @@ -359,7 +359,7 @@ const ( // VoiceAudioFormat describes a PCM audio stream format (e.g. audio/pcm @ 24 kHz). type VoiceAudioFormat struct { Type string `json:"type"` - Rate int `json:"rate"` + Rate *int `json:"rate,omitempty"` } // VoiceTurnDetection configures server-side voice-activity detection so the @@ -369,26 +369,47 @@ type VoiceTurnDetection struct { Threshold *float64 `json:"threshold,omitempty"` PrefixPaddingMs *int `json:"prefix_padding_ms,omitempty"` SilenceDurationMs *int `json:"silence_duration_ms,omitempty"` + CreateResponse *bool `json:"create_response,omitempty"` + Eagerness *string `json:"eagerness,omitempty"` + SpeechDurationMs *int `json:"speech_duration_ms,omitempty"` + RemoveFillerWords *bool `json:"remove_filler_words,omitempty"` + InterruptResponse *bool `json:"interrupt_response,omitempty"` + Languages []string `json:"languages,omitempty"` + AutoTruncate *bool `json:"auto_truncate,omitempty"` } // VoiceTranscription enables user-speech transcription events on the input stream. type VoiceTranscription struct { - Model string `json:"model,omitempty"` + Model string `json:"model,omitempty"` + Language *string `json:"language,omitempty"` + Prompt *string `json:"prompt,omitempty"` +} + +// VoiceNoiseReduction configures input audio noise reduction. +type VoiceNoiseReduction struct { + Type string `json:"type"` } // VoiceInputConfig is the input (caller -> agent) audio configuration. type VoiceInputConfig struct { - Format *VoiceAudioFormat `json:"format,omitempty"` - TurnDetection *VoiceTurnDetection `json:"turn_detection,omitempty"` - Transcription *VoiceTranscription `json:"transcription,omitempty"` + Format *VoiceAudioFormat `json:"format,omitempty"` + NoiseReduction *VoiceNoiseReduction `json:"noise_reduction,omitempty"` + EchoCancellation map[string]any `json:"echo_cancellation,omitempty"` + TurnDetection *VoiceTurnDetection `json:"turn_detection,omitempty"` + Transcription *VoiceTranscription `json:"transcription,omitempty"` } // VoiceConfig selects the output voice. Type is "openai" for realtime voices // (single lowercase word, e.g. "alloy") or "azure_standard" for Azure Neural // voices (e.g. "en-US-Ava:DragonHDLatestNeural"). type VoiceConfig struct { - Type string `json:"type"` - Name string `json:"name"` + Type string `json:"type"` + Name string `json:"name"` + Style *string `json:"style,omitempty"` + Pitch *string `json:"pitch,omitempty"` + Rate *string `json:"rate,omitempty"` + Locale *string `json:"locale,omitempty"` + Volume *string `json:"volume,omitempty"` } // VoiceOutputConfig is the output (agent -> caller) audio configuration for the @@ -399,6 +420,11 @@ type VoiceOutputConfig struct { Voice string `json:"voice,omitempty"` VoiceType string `json:"voice_type,omitempty"` VoiceLocale string `json:"voice_locale,omitempty"` + Style *string `json:"style,omitempty"` + Pitch *string `json:"pitch,omitempty"` + Rate *string `json:"rate,omitempty"` + Volume *string `json:"volume,omitempty"` + Speed *float64 `json:"speed,omitempty"` } // VoiceAudioConfig bundles the input and output audio configuration. @@ -411,12 +437,21 @@ type VoiceAudioConfig struct { // prompt voice agent. Its Kind is always AgentKindVoice ("voice"). type VoiceAgentDefinition struct { AgentDefinition - ModelType VoiceModelType `json:"model_type"` - Model string `json:"model"` - Instructions string `json:"instructions,omitempty"` - Audio *VoiceAudioConfig `json:"audio,omitempty"` - OutputModalities []string `json:"output_modalities,omitempty"` - Store *bool `json:"store,omitempty"` + ModelType VoiceModelType `json:"model_type"` + Model string `json:"model"` + Instructions string `json:"instructions,omitempty"` + StructuredInputs map[string]any `json:"structured_inputs,omitempty"` + Audio *VoiceAudioConfig `json:"audio,omitempty"` + OutputModalities []string `json:"output_modalities,omitempty"` + Store *bool `json:"store,omitempty"` + Tools []map[string]any `json:"tools,omitempty"` + Avatar map[string]any `json:"avatar,omitempty"` + Greeting map[string]any `json:"greeting,omitempty"` + Handoff map[string]any `json:"handoff,omitempty"` + ToolChoice any `json:"tool_choice,omitempty"` + ParallelToolCalls *bool `json:"parallel_tool_calls,omitempty"` + MaxOutputTokens any `json:"max_output_tokens,omitempty"` + Include []string `json:"include,omitempty"` } // CreateAgentVersionRequest represents a request to create an agent version diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go index cff9c5affe9..cc5549a588c 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go @@ -604,6 +604,114 @@ func voiceWireLocale(voice *agent_api.VoiceConfig) string { return match[1] } +func defaultVoiceAudioFormat() *agent_api.VoiceAudioFormat { + rate := defaultVoiceAudioRate + return &agent_api.VoiceAudioFormat{Type: defaultVoiceAudioType, Rate: &rate} +} + +func mapVoiceAudioFormat(format *VoiceAudioFormat, fallback *agent_api.VoiceAudioFormat) *agent_api.VoiceAudioFormat { + out := &agent_api.VoiceAudioFormat{} + if fallback != nil { + *out = *fallback + } + if format != nil { + if strings.TrimSpace(format.Type) != "" { + out.Type = strings.TrimSpace(format.Type) + } + if format.Rate != nil { + out.Rate = format.Rate + } + } + return out +} + +func mapVoiceTurnDetection(turnDetection *VoiceTurnDetection) *agent_api.VoiceTurnDetection { + out := &agent_api.VoiceTurnDetection{Type: defaultVoiceTurnDetectionType} + if turnDetection == nil { + return out + } + if strings.TrimSpace(turnDetection.Type) != "" { + out.Type = strings.TrimSpace(turnDetection.Type) + } + out.Threshold = turnDetection.Threshold + out.PrefixPaddingMs = turnDetection.PrefixPaddingMs + out.SilenceDurationMs = turnDetection.SilenceDurationMs + out.CreateResponse = turnDetection.CreateResponse + out.Eagerness = turnDetection.Eagerness + out.SpeechDurationMs = turnDetection.SpeechDurationMs + out.RemoveFillerWords = turnDetection.RemoveFillerWords + out.InterruptResponse = turnDetection.InterruptResponse + out.Languages = turnDetection.Languages + out.AutoTruncate = turnDetection.AutoTruncate + return out +} + +func mapVoiceTranscription(transcription *VoiceTranscription) *agent_api.VoiceTranscription { + out := &agent_api.VoiceTranscription{Model: defaultVoiceInputTranscriptionModel} + if transcription == nil { + return out + } + if strings.TrimSpace(transcription.Model) != "" { + out.Model = strings.TrimSpace(transcription.Model) + } + out.Language = transcription.Language + out.Prompt = transcription.Prompt + return out +} + +func mapVoiceConfig(voice *VoiceConfig, fallbackName string) *agent_api.VoiceConfig { + if voice == nil { + return buildVoiceConfig(fallbackName) + } + name := strings.TrimSpace(voice.Name) + if name == "" { + name = fallbackName + } + voiceType := strings.TrimSpace(voice.Type) + if voiceType == "" { + out := buildVoiceConfig(name) + out.Style = voice.Style + out.Pitch = voice.Pitch + out.Rate = voice.Rate + out.Locale = voice.Locale + out.Volume = voice.Volume + return out + } + return &agent_api.VoiceConfig{ + Type: voiceType, + Name: name, + Style: voice.Style, + Pitch: voice.Pitch, + Rate: voice.Rate, + Locale: voice.Locale, + Volume: voice.Volume, + } +} + +func mapVoiceStructuredInputs(inputs map[string]any) map[string]any { + if len(inputs) == 0 { + return nil + } + out := make(map[string]any, len(inputs)) + for name, input := range inputs { + inputMap, ok := input.(map[string]any) + if !ok { + out[name] = input + continue + } + + mapped := maps.Clone(inputMap) + if value, ok := mapped["defaultValue"]; ok { + if _, hasSnakeCase := mapped["default_value"]; !hasSnakeCase { + mapped["default_value"] = value + } + delete(mapped, "defaultValue") + } + out[name] = mapped + } + return out +} + // CreateVoiceAgentAPIRequest builds a CreateAgentRequest for a declarative // voice agent. It translates the authoring kind "prompt-voice" into the // data-plane service kind "voice" and defaults the audio pipeline. @@ -640,36 +748,76 @@ func createVoiceAgentAPIRequest(voiceAgent VoiceAgent) (*agent_api.CreateAgentRe voiceName = *voiceAgent.Voice } - audioFormat := &agent_api.VoiceAudioFormat{ - Type: defaultVoiceAudioType, - Rate: defaultVoiceAudioRate, + inputFormat := defaultVoiceAudioFormat() + outputFormat := defaultVoiceAudioFormat() + turnDetection := mapVoiceTurnDetection(nil) + transcription := mapVoiceTranscription(nil) + var noiseReduction *agent_api.VoiceNoiseReduction + var echoCancellation map[string]any + outputVoice := buildVoiceConfig(voiceName) + var outputSpeed *float64 + if voiceAgent.Audio != nil { + if voiceAgent.Audio.Input != nil { + inputFormat = mapVoiceAudioFormat(voiceAgent.Audio.Input.Format, inputFormat) + if voiceAgent.Audio.Input.NoiseReduction != nil { + noiseReduction = &agent_api.VoiceNoiseReduction{Type: strings.TrimSpace(voiceAgent.Audio.Input.NoiseReduction.Type)} + } + echoCancellation = voiceAgent.Audio.Input.EchoCancellation + turnDetection = mapVoiceTurnDetection(voiceAgent.Audio.Input.TurnDetection) + transcription = mapVoiceTranscription(voiceAgent.Audio.Input.Transcription) + } + if voiceAgent.Audio.Output != nil { + outputFormat = mapVoiceAudioFormat(voiceAgent.Audio.Output.Format, outputFormat) + outputVoice = mapVoiceConfig(voiceAgent.Audio.Output.Voice, voiceName) + outputSpeed = voiceAgent.Audio.Output.Speed + } + } + + outputModalities := []string{"audio"} + if len(voiceAgent.OutputModalities) > 0 { + outputModalities = voiceAgent.OutputModalities } input := &agent_api.VoiceInputConfig{ - Format: audioFormat, - TurnDetection: &agent_api.VoiceTurnDetection{Type: defaultVoiceTurnDetectionType}, - Transcription: &agent_api.VoiceTranscription{Model: defaultVoiceInputTranscriptionModel}, + Format: inputFormat, + NoiseReduction: noiseReduction, + EchoCancellation: echoCancellation, + TurnDetection: turnDetection, + Transcription: transcription, } - voiceConfig := buildVoiceConfig(voiceName) voiceDef := agent_api.VoiceAgentDefinition{ AgentDefinition: agent_api.AgentDefinition{ // Translate authoring kind prompt-voice -> service kind voice. Kind: agent_api.AgentKindVoice, }, - ModelType: modelType, - Model: modelID, - Instructions: instructions, + ModelType: modelType, + Model: modelID, + Instructions: instructions, + StructuredInputs: mapVoiceStructuredInputs(voiceAgent.StructuredInputs), Audio: &agent_api.VoiceAudioConfig{ Input: input, Output: &agent_api.VoiceOutputConfig{ - Format: audioFormat, - Voice: voiceConfig.Name, - VoiceType: voiceWireType(voiceConfig), - VoiceLocale: voiceWireLocale(voiceConfig), + Format: outputFormat, + Voice: outputVoice.Name, + VoiceType: voiceWireType(outputVoice), + VoiceLocale: voiceWireLocale(outputVoice), + Style: outputVoice.Style, + Pitch: outputVoice.Pitch, + Rate: outputVoice.Rate, + Volume: outputVoice.Volume, + Speed: outputSpeed, }, }, - OutputModalities: []string{"audio"}, - Store: voiceAgent.Store, + OutputModalities: outputModalities, + Store: voiceAgent.Store, + Tools: voiceAgent.Tools, + Avatar: voiceAgent.Avatar, + Greeting: voiceAgent.Greeting, + Handoff: voiceAgent.Handoff, + ToolChoice: voiceAgent.ToolChoice, + ParallelToolCalls: voiceAgent.ParallelToolCalls, + MaxOutputTokens: voiceAgent.MaxOutputTokens, + Include: voiceAgent.Include, } return createAgentAPIRequest(voiceAgent.AgentDefinition, voiceDef, nil, nil) diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go index 4b603303bfb..b507e68b377 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go @@ -10,6 +10,8 @@ import ( "azureaiagent/internal/pkg/agents/agent_api" ) +func ptr[T any](v T) *T { return &v } + // --------------------------------------------------------------------------- // isOpenAIVoice / buildVoiceConfig // --------------------------------------------------------------------------- @@ -118,7 +120,8 @@ func TestCreateVoiceAgentAPIRequest_Defaults(t *testing.T) { t.Fatalf("Audio pipeline not populated: %+v", def.Audio) } in := def.Audio.Input - if in.Format == nil || in.Format.Type != defaultVoiceAudioType || in.Format.Rate != defaultVoiceAudioRate { + if in.Format == nil || in.Format.Type != defaultVoiceAudioType || in.Format.Rate == nil || + *in.Format.Rate != defaultVoiceAudioRate { t.Errorf("input format = %+v", in.Format) } if in.TurnDetection == nil || in.TurnDetection.Type != defaultVoiceTurnDetectionType { @@ -128,7 +131,8 @@ func TestCreateVoiceAgentAPIRequest_Defaults(t *testing.T) { t.Errorf("transcription = %+v", in.Transcription) } out := def.Audio.Output - if out.Format == nil || out.Format.Type != defaultVoiceAudioType || out.Format.Rate != defaultVoiceAudioRate { + if out.Format == nil || out.Format.Type != defaultVoiceAudioType || out.Format.Rate == nil || + *out.Format.Rate != defaultVoiceAudioRate { t.Errorf("output format = %+v", out.Format) } // Default voice is the DragonHD Azure Neural voice in the flat unified shape. @@ -309,6 +313,100 @@ func TestCreateVoiceAgentAPIRequest_MarshalServiceWireShape(t *testing.T) { } } +func TestCreateVoiceAgentAPIRequest_AdvancedSettingsWireShape(t *testing.T) { + t.Parallel() + inRate := 16000 + outRate := 24000 + threshold := 0.6 + speechDurationMs := 120 + createResponse := true + removeFillerWords := true + interruptResponse := true + autoTruncate := true + speed := 1.1 + parallelToolCalls := true + style := "cheerful" + pitch := "+0Hz" + rate := "+0%" + volume := "+0%" + agent := VoiceAgent{ + AgentDefinition: AgentDefinition{Kind: AgentKindPromptVoice, Name: "voice-advanced"}, + Model: &Model{Id: "gpt-realtime"}, + Instructions: ptr("You are {{persona}}, a concise voice assistant."), + StructuredInputs: map[string]any{ + "persona": map[string]any{"description": "Assistant persona", "defaultValue": "Ada"}, + }, + Audio: &VoiceAudio{ + Input: &VoiceAudioInput{ + Format: &VoiceAudioFormat{Type: "audio/pcmu", Rate: &inRate}, + NoiseReduction: &VoiceNoiseReduction{Type: "near_field"}, + EchoCancellation: map[string]any{"type": "server_echo_cancellation", "channels": 1}, + TurnDetection: &VoiceTurnDetection{ + Type: "azure_semantic_vad", + Threshold: &threshold, + SpeechDurationMs: &speechDurationMs, + CreateResponse: &createResponse, + RemoveFillerWords: &removeFillerWords, + InterruptResponse: &interruptResponse, + Languages: []string{"en-US"}, + AutoTruncate: &autoTruncate, + }, + Transcription: &VoiceTranscription{Model: "whisper-1", Language: ptr("en-US"), Prompt: ptr("Contoso terms")}, + }, + Output: &VoiceAudioOutput{ + Format: &VoiceAudioFormat{Type: "audio/pcm", Rate: &outRate}, + Voice: &VoiceConfig{ + Type: "azure_standard", Name: "en-US-AvaNeural", Style: &style, + Pitch: &pitch, Rate: &rate, Locale: ptr("en-US"), Volume: &volume, + }, + Speed: &speed, + }, + }, + OutputModalities: []string{"audio", "text"}, + Tools: []map[string]any{{"type": "system", "name": "end_conversation"}}, + Avatar: map[string]any{"type": "video_avatar", "character": "lisa", "output_protocol": "webrtc"}, + Greeting: map[string]any{"type": "template", "text": "Hello {{persona}}"}, + ToolChoice: "auto", + ParallelToolCalls: ¶llelToolCalls, + MaxOutputTokens: "inf", + Include: []string{"item.input_audio_transcription.phrases"}, + } + + req, err := CreateVoiceAgentAPIRequest(agent) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + payload, err := json.Marshal(req) + if err != nil { + t.Fatalf("marshal request: %v", err) + } + + var wire map[string]any + if err := json.Unmarshal(payload, &wire); err != nil { + t.Fatalf("unmarshal payload: %v", err) + } + def := wire["definition"].(map[string]any) + input := def["audio"].(map[string]any)["input"].(map[string]any) + output := def["audio"].(map[string]any)["output"].(map[string]any) + structured := def["structured_inputs"].(map[string]any)["persona"].(map[string]any) + + if structured["default_value"] != "Ada" || structured["defaultValue"] != nil { + t.Fatalf("structured input default was not mapped to wire shape: %#v", structured) + } + if output["voice"] != "en-US-AvaNeural" || output["voice_type"] != "azure-standard" || output["style"] != style { + t.Fatalf("output voice flat shape not mapped: %#v", output) + } + if input["echo_cancellation"].(map[string]any)["type"] != "server_echo_cancellation" { + t.Fatalf("echo cancellation not mapped: %#v", input["echo_cancellation"]) + } + if def["tool_choice"] != "auto" || def["max_output_tokens"] != "inf" { + t.Fatalf("response options not mapped: %#v", def) + } + if len(def["tools"].([]any)) != 1 || def["avatar"].(map[string]any)["character"] != "lisa" { + t.Fatalf("tools/avatar not mapped: %#v", def) + } +} + // TestCreateVoiceAgentAPIRequest_ExplicitManaged verifies that explicitly // setting model_type: managed is accepted (idempotent with the default). func TestCreateVoiceAgentAPIRequest_ExplicitManaged(t *testing.T) { diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go index 996eae8a561..a90fbe6046d 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go @@ -460,6 +460,7 @@ func ValidateAgentDefinition(templateBytes []byte) error { "template.model_type '%s' is not supported; use '%s' or '%s'", agent.ModelType, VoiceModelTypeManaged, VoiceModelTypeSelfDeployed)) } + errors = append(errors, validateVoiceAgentAdvancedConfig(agent)...) } else { errors = append(errors, fmt.Sprintf("failed to unmarshal to VoiceAgent: %v", err)) } @@ -479,6 +480,94 @@ func ValidateAgentDefinition(templateBytes []byte) error { return nil } +func validateVoiceAgentAdvancedConfig(agent VoiceAgent) []string { + var errors []string + for i, modality := range agent.OutputModalities { + if strings.TrimSpace(modality) == "" { + errors = append(errors, fmt.Sprintf("template.output_modalities[%d] must not be blank", i)) + } + } + + if agent.Audio == nil { + return append(errors, validateVoiceIncludeTranscriptionCompatibility(agent, "")...) + } + transcriptionModel := "" + if agent.Audio.Input != nil { + errors = append(errors, validateVoiceAudioFormat("template.audio.input.format", agent.Audio.Input.Format)...) + if nr := agent.Audio.Input.NoiseReduction; nr != nil && strings.TrimSpace(nr.Type) == "" { + errors = append(errors, "template.audio.input.noise_reduction.type must not be blank") + } + if td := agent.Audio.Input.TurnDetection; td != nil { + if strings.TrimSpace(td.Type) == "" { + errors = append(errors, "template.audio.input.turn_detection.type must not be blank") + } + if td.Threshold != nil && (*td.Threshold < 0 || *td.Threshold > 1) { + errors = append(errors, "template.audio.input.turn_detection.threshold must be between 0 and 1") + } + if td.PrefixPaddingMs != nil && *td.PrefixPaddingMs < 0 { + errors = append(errors, "template.audio.input.turn_detection.prefix_padding_ms must be >= 0") + } + if td.SilenceDurationMs != nil && *td.SilenceDurationMs < 0 { + errors = append(errors, "template.audio.input.turn_detection.silence_duration_ms must be >= 0") + } + if td.SpeechDurationMs != nil && *td.SpeechDurationMs < 0 { + errors = append(errors, "template.audio.input.turn_detection.speech_duration_ms must be >= 0") + } + } + if agent.Audio.Input.Transcription != nil { + transcriptionModel = agent.Audio.Input.Transcription.Model + } + } + if agent.Audio.Output != nil { + errors = append(errors, validateVoiceAudioFormat("template.audio.output.format", agent.Audio.Output.Format)...) + if voice := agent.Audio.Output.Voice; voice != nil { + if strings.TrimSpace(voice.Type) == "" { + errors = append(errors, "template.audio.output.voice.type must not be blank") + } + if strings.TrimSpace(voice.Name) == "" { + errors = append(errors, "template.audio.output.voice.name must not be blank") + } + } + if speed := agent.Audio.Output.Speed; speed != nil && (*speed < 0.25 || *speed > 1.5) { + errors = append(errors, "template.audio.output.speed must be between 0.25 and 1.5") + } + } + return append(errors, validateVoiceIncludeTranscriptionCompatibility(agent, transcriptionModel)...) +} + +func validateVoiceIncludeTranscriptionCompatibility(agent VoiceAgent, transcriptionModel string) []string { + if !slices.Contains(agent.Include, "item.input_audio_transcription.phrases") { + return nil + } + model := strings.TrimSpace(transcriptionModel) + if model == "" { + model = defaultVoiceInputTranscriptionModel + } + if model == "azure-speech" || model == "azure-fast-transcription" { + return nil + } + return []string{ + "template.include item.input_audio_transcription.phrases requires template.audio.input.transcription.model to be azure-speech or azure-fast-transcription", + } +} + +func validateVoiceAudioFormat(path string, format *VoiceAudioFormat) []string { + if format == nil { + return nil + } + var errors []string + formatType := strings.TrimSpace(format.Type) + if formatType == "" { + errors = append(errors, path+".type must not be blank") + } else if formatType != "audio/pcm" && formatType != "audio/pcmu" && formatType != "audio/pcma" { + errors = append(errors, path+".type must be 'audio/pcm', 'audio/pcmu', or 'audio/pcma'") + } + if format.Rate != nil && *format.Rate <= 0 { + errors = append(errors, path+".rate must be greater than 0") + } + return errors +} + // Validate that the agent name matches the expected deployable format func ValidateAgentName(name string) error { if name == "" { diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/yaml.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/yaml.go index 46fffca9e6a..ffa35d057db 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/yaml.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/yaml.go @@ -214,9 +214,98 @@ type VoiceAgent struct { // Voice is the output voice name (e.g. "en-US-Ava:DragonHDLatestNeural" for // an Azure Neural voice, or "alloy" for an OpenAI realtime voice). Voice *string `json:"voice,omitempty" yaml:"voice,omitempty"` + // StructuredInputs declares template inputs used by voice instructions and greeting. + StructuredInputs map[string]any `json:"structuredInputs,omitempty" yaml:"structured_inputs,omitempty"` + // Audio customizes the input and output voice pipeline. Missing fields keep azd defaults. + Audio *VoiceAudio `json:"audio,omitempty" yaml:"audio,omitempty"` + // OutputModalities declares response modalities such as audio, text, animation, or avatar. + OutputModalities []string `json:"outputModalities,omitempty" yaml:"output_modalities,omitempty"` // Store toggles server-side logging (transcript + per-turn audio). Optional; // the service defaults to false when omitted. Store *bool `json:"store,omitempty" yaml:"store,omitempty"` + // Tools are passed through to the prompt voice service. Supported direct tool + // types include function, mcp, system, and toolbox. + Tools []map[string]any `json:"tools,omitempty" yaml:"tools,omitempty"` + // Avatar customizes voice avatar output for services that support it. + Avatar map[string]any `json:"avatar,omitempty" yaml:"avatar,omitempty"` + // Greeting configures initial greeting behavior for services that support it. + Greeting map[string]any `json:"greeting,omitempty" yaml:"greeting,omitempty"` + // Handoff configures voice handoff behavior for services that support it. + Handoff map[string]any `json:"handoff,omitempty" yaml:"handoff,omitempty"` + // ToolChoice configures service tool choice behavior, such as auto/none/required. + ToolChoice any `json:"toolChoice,omitempty" yaml:"tool_choice,omitempty"` + // ParallelToolCalls toggles parallel tool calls. + ParallelToolCalls *bool `json:"parallelToolCalls,omitempty" yaml:"parallel_tool_calls,omitempty"` + // MaxOutputTokens limits response output tokens. Use an integer or service-supported string such as "inf". + MaxOutputTokens any `json:"maxOutputTokens,omitempty" yaml:"max_output_tokens,omitempty"` + // Include requests additional service response fields. + Include []string `json:"include,omitempty" yaml:"include,omitempty"` +} + +// VoiceAudio bundles optional prompt voice input/output audio overrides. +type VoiceAudio struct { + Input *VoiceAudioInput `json:"input,omitempty" yaml:"input,omitempty"` + Output *VoiceAudioOutput `json:"output,omitempty" yaml:"output,omitempty"` +} + +// VoiceAudioInput customizes caller-to-agent audio. +type VoiceAudioInput struct { + Format *VoiceAudioFormat `json:"format,omitempty" yaml:"format,omitempty"` + NoiseReduction *VoiceNoiseReduction `json:"noiseReduction,omitempty" yaml:"noise_reduction,omitempty"` + EchoCancellation map[string]any `json:"echoCancellation,omitempty" yaml:"echo_cancellation,omitempty"` + TurnDetection *VoiceTurnDetection `json:"turnDetection,omitempty" yaml:"turn_detection,omitempty"` + Transcription *VoiceTranscription `json:"transcription,omitempty" yaml:"transcription,omitempty"` +} + +// VoiceAudioOutput customizes agent-to-caller audio. +type VoiceAudioOutput struct { + Format *VoiceAudioFormat `json:"format,omitempty" yaml:"format,omitempty"` + Voice *VoiceConfig `json:"voice,omitempty" yaml:"voice,omitempty"` + Speed *float64 `json:"speed,omitempty" yaml:"speed,omitempty"` +} + +// VoiceAudioFormat describes an audio stream format. +type VoiceAudioFormat struct { + Type string `json:"type" yaml:"type"` + Rate *int `json:"rate,omitempty" yaml:"rate,omitempty"` +} + +// VoiceNoiseReduction configures input audio noise reduction. +type VoiceNoiseReduction struct { + Type string `json:"type" yaml:"type"` +} + +// VoiceTurnDetection configures server-side turn detection. +type VoiceTurnDetection struct { + Type string `json:"type" yaml:"type"` + Threshold *float64 `json:"threshold,omitempty" yaml:"threshold,omitempty"` + PrefixPaddingMs *int `json:"prefixPaddingMs,omitempty" yaml:"prefix_padding_ms,omitempty"` + SilenceDurationMs *int `json:"silenceDurationMs,omitempty" yaml:"silence_duration_ms,omitempty"` + CreateResponse *bool `json:"createResponse,omitempty" yaml:"create_response,omitempty"` + Eagerness *string `json:"eagerness,omitempty" yaml:"eagerness,omitempty"` + SpeechDurationMs *int `json:"speechDurationMs,omitempty" yaml:"speech_duration_ms,omitempty"` + RemoveFillerWords *bool `json:"removeFillerWords,omitempty" yaml:"remove_filler_words,omitempty"` + InterruptResponse *bool `json:"interruptResponse,omitempty" yaml:"interrupt_response,omitempty"` + Languages []string `json:"languages,omitempty" yaml:"languages,omitempty"` + AutoTruncate *bool `json:"autoTruncate,omitempty" yaml:"auto_truncate,omitempty"` +} + +// VoiceTranscription configures input transcription. +type VoiceTranscription struct { + Model string `json:"model,omitempty" yaml:"model,omitempty"` + Language *string `json:"language,omitempty" yaml:"language,omitempty"` + Prompt *string `json:"prompt,omitempty" yaml:"prompt,omitempty"` +} + +// VoiceConfig selects the output voice. +type VoiceConfig struct { + Type string `json:"type" yaml:"type"` + Name string `json:"name" yaml:"name"` + Style *string `json:"style,omitempty" yaml:"style,omitempty"` + Pitch *string `json:"pitch,omitempty" yaml:"pitch,omitempty"` + Rate *string `json:"rate,omitempty" yaml:"rate,omitempty"` + Locale *string `json:"locale,omitempty" yaml:"locale,omitempty"` + Volume *string `json:"volume,omitempty" yaml:"volume,omitempty"` } // ContainerResources represents the resource allocation for a containerized agent. diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/agent_definition.go b/cli/azd/extensions/azure.ai.agents/internal/project/agent_definition.go index 26b5a06f874..1625d9a7637 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/agent_definition.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/agent_definition.go @@ -138,35 +138,68 @@ type AgentDefinitionInline struct { // Voice-agent fields (kind: prompt-voice). All omitempty so container/ // workflow entries are byte-for-byte unchanged. - ModelType agent_yaml.VoiceModelType `json:"modelType,omitempty"` - Model *agent_yaml.Model `json:"model,omitempty"` - Instructions *string `json:"instructions,omitempty"` - Voice *string `json:"voice,omitempty"` - Store *bool `json:"store,omitempty"` + ModelType agent_yaml.VoiceModelType `json:"modelType,omitempty"` + Model *agent_yaml.Model `json:"model,omitempty"` + Instructions *string `json:"instructions,omitempty"` + Voice *string `json:"voice,omitempty"` + StructuredInputs map[string]any `json:"structuredInputs,omitempty"` + Audio *agent_yaml.VoiceAudio `json:"audio,omitempty"` + OutputModalities []string `json:"outputModalities,omitempty"` + Store *bool `json:"store,omitempty"` + Tools []map[string]any `json:"tools,omitempty"` + Avatar map[string]any `json:"avatar,omitempty"` + Greeting map[string]any `json:"greeting,omitempty"` + Handoff map[string]any `json:"handoff,omitempty"` + ToolChoice any `json:"toolChoice,omitempty"` + ParallelToolCalls *bool `json:"parallelToolCalls,omitempty"` + MaxOutputTokens any `json:"maxOutputTokens,omitempty"` + Include []string `json:"include,omitempty"` } // voiceAgentDefinitionToInline projects a VoiceAgent into the inline definition // written to azure.yaml. Voice agents carry no container/image/code config. func voiceAgentDefinitionToInline(va agent_yaml.VoiceAgent) AgentDefinitionInline { return AgentDefinitionInline{ - AgentDefinition: va.AgentDefinition, - ModelType: va.ModelType, - Model: va.Model, - Instructions: va.Instructions, - Voice: va.Voice, - Store: va.Store, + AgentDefinition: va.AgentDefinition, + ModelType: va.ModelType, + Model: va.Model, + Instructions: va.Instructions, + Voice: va.Voice, + StructuredInputs: va.StructuredInputs, + Audio: va.Audio, + OutputModalities: va.OutputModalities, + Store: va.Store, + Tools: va.Tools, + Avatar: va.Avatar, + Greeting: va.Greeting, + Handoff: va.Handoff, + ToolChoice: va.ToolChoice, + ParallelToolCalls: va.ParallelToolCalls, + MaxOutputTokens: va.MaxOutputTokens, + Include: va.Include, } } // toVoiceAgent rebuilds an agent_yaml.VoiceAgent from the inline definition. func (d AgentDefinitionInline) toVoiceAgent() agent_yaml.VoiceAgent { return agent_yaml.VoiceAgent{ - AgentDefinition: d.AgentDefinition, - ModelType: d.ModelType, - Model: d.Model, - Instructions: d.Instructions, - Voice: d.Voice, - Store: d.Store, + AgentDefinition: d.AgentDefinition, + ModelType: d.ModelType, + Model: d.Model, + Instructions: d.Instructions, + Voice: d.Voice, + StructuredInputs: d.StructuredInputs, + Audio: d.Audio, + OutputModalities: d.OutputModalities, + Store: d.Store, + Tools: d.Tools, + Avatar: d.Avatar, + Greeting: d.Greeting, + Handoff: d.Handoff, + ToolChoice: d.ToolChoice, + ParallelToolCalls: d.ParallelToolCalls, + MaxOutputTokens: d.MaxOutputTokens, + Include: d.Include, } } diff --git a/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json b/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json index 553e7692fd2..d8ad5e104bf 100644 --- a/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json +++ b/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json @@ -75,6 +75,54 @@ "type": "boolean", "description": "Voice agent (kind: prompt-voice) server-side logging toggle (transcript + per-turn audio). Defaults to false when omitted." }, + "structuredInputs": { + "type": "object", + "description": "Voice agent (kind: prompt-voice) structured prompt inputs. Use description, defaultValue, schema, and required; azd maps defaultValue to the service wire field default_value.", + "additionalProperties": true + }, + "audio": { + "$ref": "#/definitions/VoiceAudio" + }, + "outputModalities": { + "type": "array", + "description": "Voice agent (kind: prompt-voice) output modalities. Well-known values are audio, text, animation, and avatar.", + "items": { "type": "string", "minLength": 1 } + }, + "tools": { + "type": "array", + "description": "Voice agent (kind: prompt-voice) tools. Direct tool types include function, mcp, system, and toolbox.", + "items": { "type": "object", "additionalProperties": true } + }, + "avatar": { + "type": "object", + "description": "Voice agent (kind: prompt-voice) avatar configuration.", + "additionalProperties": true + }, + "greeting": { + "type": "object", + "description": "Voice agent (kind: prompt-voice) greeting configuration, such as template or llm_generated.", + "additionalProperties": true + }, + "handoff": { + "type": "object", + "description": "Voice agent (kind: prompt-voice) handoff configuration.", + "additionalProperties": true + }, + "toolChoice": { + "description": "Voice agent (kind: prompt-voice) tool choice behavior, such as none, auto, required, or a tool choice object." + }, + "parallelToolCalls": { + "type": "boolean", + "description": "Voice agent (kind: prompt-voice) parallel tool call toggle. Currently rejected by azd for prompt voice runtime." + }, + "maxOutputTokens": { + "description": "Voice agent (kind: prompt-voice) maximum output tokens. Use an integer or a service-supported string such as inf." + }, + "include": { + "type": "array", + "description": "Voice agent (kind: prompt-voice) extra service response fields to include.", + "items": { "type": "string" } + }, "name": { "type": "string", "description": "The agent name." @@ -235,6 +283,93 @@ "required": ["runtime", "entryPoint"], "additionalProperties": false }, + "VoiceAudio": { + "type": "object", + "description": "Prompt voice input and output audio configuration.", + "properties": { + "input": { "$ref": "#/definitions/VoiceAudioInput" }, + "output": { "$ref": "#/definitions/VoiceAudioOutput" } + }, + "additionalProperties": false + }, + "VoiceAudioInput": { + "type": "object", + "properties": { + "format": { "$ref": "#/definitions/VoiceAudioFormat" }, + "noiseReduction": { "$ref": "#/definitions/VoiceNoiseReduction" }, + "echoCancellation": { "type": "object", "additionalProperties": true }, + "turnDetection": { "$ref": "#/definitions/VoiceTurnDetection" }, + "transcription": { "$ref": "#/definitions/VoiceTranscription" } + }, + "additionalProperties": false + }, + "VoiceAudioOutput": { + "type": "object", + "properties": { + "format": { "$ref": "#/definitions/VoiceAudioFormat" }, + "voice": { "$ref": "#/definitions/VoiceConfig" }, + "speed": { "type": "number", "minimum": 0.25, "maximum": 1.5 } + }, + "additionalProperties": false + }, + "VoiceAudioFormat": { + "type": "object", + "properties": { + "type": { "type": "string", "enum": ["audio/pcm", "audio/pcmu", "audio/pcma"] }, + "rate": { "type": "integer", "minimum": 1 } + }, + "required": ["type"], + "additionalProperties": false + }, + "VoiceNoiseReduction": { + "type": "object", + "properties": { + "type": { "type": "string", "description": "Well-known values include near_field, far_field, and azure_deep_noise_suppression." } + }, + "required": ["type"], + "additionalProperties": false + }, + "VoiceTurnDetection": { + "type": "object", + "properties": { + "type": { "type": "string", "description": "Well-known values include server_vad, semantic_vad, and azure_semantic_vad." }, + "threshold": { "type": "number", "minimum": 0, "maximum": 1 }, + "prefixPaddingMs": { "type": "integer", "minimum": 0 }, + "silenceDurationMs": { "type": "integer", "minimum": 0 }, + "createResponse": { "type": "boolean" }, + "eagerness": { "type": "string" }, + "speechDurationMs": { "type": "integer", "minimum": 0 }, + "removeFillerWords": { "type": "boolean" }, + "interruptResponse": { "type": "boolean" }, + "languages": { "type": "array", "items": { "type": "string" } }, + "autoTruncate": { "type": "boolean" } + }, + "required": ["type"], + "additionalProperties": false + }, + "VoiceTranscription": { + "type": "object", + "properties": { + "model": { "type": "string" }, + "language": { "type": "string" }, + "prompt": { "type": "string" } + }, + "additionalProperties": false + }, + "VoiceConfig": { + "type": "object", + "properties": { + "type": { "type": "string", "description": "Voice provider type, such as openai, azure_standard, or azure-standard." }, + "name": { "type": "string" }, + "style": { "type": "string" }, + "pitch": { "type": "string" }, + "rate": { "type": "string" }, + "locale": { "type": "string" }, + "volume": { "type": "string" } + }, + "required": ["type", "name"], + "additionalProperties": false + }, "SessionConfiguration": { "type": "object", "description": "Optional hosted-agent session runtime settings. When omitted, the service applies its defaults (idle timeout 900 seconds).", From ab7851f759d96a810b0cdc4dc9dd78fd546a8c44 Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Tue, 25 Aug 2026 19:53:47 +0800 Subject: [PATCH 30/38] fix(agents): reject unsupported voice parallel tools --- .../internal/pkg/agents/agent_yaml/parse.go | 4 +++ .../pkg/agents/agent_yaml/parse_voice_test.go | 36 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go index a90fbe6046d..e7435c054c3 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go @@ -487,6 +487,10 @@ func validateVoiceAgentAdvancedConfig(agent VoiceAgent) []string { errors = append(errors, fmt.Sprintf("template.output_modalities[%d] must not be blank", i)) } } + if agent.ParallelToolCalls != nil { + errors = append(errors, + "template.parallel_tool_calls is not currently supported by the prompt voice runtime; remove it from azure.yaml") + } if agent.Audio == nil { return append(errors, validateVoiceIncludeTranscriptionCompatibility(agent, "")...) diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse_voice_test.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse_voice_test.go index 6aba8a368dd..ded2d4e520c 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse_voice_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse_voice_test.go @@ -118,3 +118,39 @@ model_type: unsupported t.Fatalf("expected invalid model_type error, got: %v", err) } } + +func TestValidateAgentDefinition_PromptVoice_RejectsParallelToolCalls(t *testing.T) { + yamlContent := []byte(` +kind: prompt-voice +name: voice-agent +model: + id: gpt-realtime +parallel_tool_calls: true +`) + err := ValidateAgentDefinition(yamlContent) + if err == nil || !strings.Contains(err.Error(), "parallel_tool_calls is not currently supported") { + t.Fatalf("expected parallel_tool_calls validation error, got: %v", err) + } +} + +func TestValidateAgentDefinition_PromptVoice_InvalidIncludeTranscriptionModel(t *testing.T) { + yamlContent := []byte(` +kind: prompt-voice +name: voice-agent +model: + id: gpt-realtime +audio: + input: + transcription: + model: whisper-1 +include: + - item.input_audio_transcription.phrases +`) + err := ValidateAgentDefinition(yamlContent) + if err == nil { + t.Fatal("expected include/transcription validation error") + } + if !strings.Contains(err.Error(), "azure-speech") || !strings.Contains(err.Error(), "azure-fast-transcription") { + t.Fatalf("expected transcription model guidance in error, got: %v", err) + } +} From 4c4120e36974327ef850556bbfb8111a145eb0bd Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Wed, 26 Aug 2026 11:38:33 +0800 Subject: [PATCH 31/38] fix(agents): validate voice runtime settings --- .../internal/pkg/agents/agent_yaml/parse.go | 4 ++-- .../pkg/agents/agent_yaml/parse_voice_test.go | 18 ++++++++++++++++++ .../schemas/azure.ai.agent.json | 2 +- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go index e7435c054c3..b6180ca4ff1 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go @@ -505,8 +505,8 @@ func validateVoiceAgentAdvancedConfig(agent VoiceAgent) []string { if strings.TrimSpace(td.Type) == "" { errors = append(errors, "template.audio.input.turn_detection.type must not be blank") } - if td.Threshold != nil && (*td.Threshold < 0 || *td.Threshold > 1) { - errors = append(errors, "template.audio.input.turn_detection.threshold must be between 0 and 1") + if td.Threshold != nil && (*td.Threshold <= 0 || *td.Threshold > 1) { + errors = append(errors, "template.audio.input.turn_detection.threshold must be greater than 0 and <= 1") } if td.PrefixPaddingMs != nil && *td.PrefixPaddingMs < 0 { errors = append(errors, "template.audio.input.turn_detection.prefix_padding_ms must be >= 0") diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse_voice_test.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse_voice_test.go index ded2d4e520c..577754832fd 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse_voice_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse_voice_test.go @@ -133,6 +133,24 @@ parallel_tool_calls: true } } +func TestValidateAgentDefinition_PromptVoice_RejectsZeroTurnDetectionThreshold(t *testing.T) { + yamlContent := []byte(` +kind: prompt-voice +name: voice-agent +model: + id: gpt-realtime +audio: + input: + turn_detection: + type: azure_semantic_vad + threshold: 0 +`) + err := ValidateAgentDefinition(yamlContent) + if err == nil || !strings.Contains(err.Error(), "threshold must be greater than 0") { + t.Fatalf("expected threshold validation error, got: %v", err) + } +} + func TestValidateAgentDefinition_PromptVoice_InvalidIncludeTranscriptionModel(t *testing.T) { yamlContent := []byte(` kind: prompt-voice diff --git a/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json b/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json index d8ad5e104bf..c5319e87ec5 100644 --- a/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json +++ b/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json @@ -333,7 +333,7 @@ "type": "object", "properties": { "type": { "type": "string", "description": "Well-known values include server_vad, semantic_vad, and azure_semantic_vad." }, - "threshold": { "type": "number", "minimum": 0, "maximum": 1 }, + "threshold": { "type": "number", "exclusiveMinimum": 0, "maximum": 1 }, "prefixPaddingMs": { "type": "integer", "minimum": 0 }, "silenceDurationMs": { "type": "integer", "minimum": 0 }, "createResponse": { "type": "boolean" }, From 79494be9391590290145aa0bf1f12c9738224b68 Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Wed, 26 Aug 2026 13:54:55 +0800 Subject: [PATCH 32/38] test(agents): remove pointer helper --- .../internal/pkg/agents/agent_yaml/map_voice_test.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go index b507e68b377..8c1d0eb6d17 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go @@ -10,8 +10,6 @@ import ( "azureaiagent/internal/pkg/agents/agent_api" ) -func ptr[T any](v T) *T { return &v } - // --------------------------------------------------------------------------- // isOpenAIVoice / buildVoiceConfig // --------------------------------------------------------------------------- @@ -332,7 +330,7 @@ func TestCreateVoiceAgentAPIRequest_AdvancedSettingsWireShape(t *testing.T) { agent := VoiceAgent{ AgentDefinition: AgentDefinition{Kind: AgentKindPromptVoice, Name: "voice-advanced"}, Model: &Model{Id: "gpt-realtime"}, - Instructions: ptr("You are {{persona}}, a concise voice assistant."), + Instructions: new("You are {{persona}}, a concise voice assistant."), StructuredInputs: map[string]any{ "persona": map[string]any{"description": "Assistant persona", "defaultValue": "Ada"}, }, @@ -351,13 +349,13 @@ func TestCreateVoiceAgentAPIRequest_AdvancedSettingsWireShape(t *testing.T) { Languages: []string{"en-US"}, AutoTruncate: &autoTruncate, }, - Transcription: &VoiceTranscription{Model: "whisper-1", Language: ptr("en-US"), Prompt: ptr("Contoso terms")}, + Transcription: &VoiceTranscription{Model: "whisper-1", Language: new("en-US"), Prompt: new("Contoso terms")}, }, Output: &VoiceAudioOutput{ Format: &VoiceAudioFormat{Type: "audio/pcm", Rate: &outRate}, Voice: &VoiceConfig{ Type: "azure_standard", Name: "en-US-AvaNeural", Style: &style, - Pitch: &pitch, Rate: &rate, Locale: ptr("en-US"), Volume: &volume, + Pitch: &pitch, Rate: &rate, Locale: new("en-US"), Volume: &volume, }, Speed: &speed, }, From f76134acf94fbc4ee734c17d254f8d306e8361c4 Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Wed, 26 Aug 2026 14:58:34 +0800 Subject: [PATCH 33/38] fix(agents): prefer explicit voice locale --- .../internal/pkg/agents/agent_yaml/map.go | 8 ++++++- .../pkg/agents/agent_yaml/map_voice_test.go | 21 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go index cc5549a588c..0e2dce82e0d 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go @@ -594,7 +594,13 @@ func voiceWireType(voice *agent_api.VoiceConfig) string { } func voiceWireLocale(voice *agent_api.VoiceConfig) string { - if voice == nil || voice.Name == "" || isOpenAIVoice(voice.Name) { + if voice == nil || voice.Name == "" { + return "" + } + if voice.Locale != nil && strings.TrimSpace(*voice.Locale) != "" { + return strings.TrimSpace(*voice.Locale) + } + if isOpenAIVoice(voice.Name) { return "" } match := azureNeuralVoicePattern.FindStringSubmatch(voice.Name) diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go index 8c1d0eb6d17..a72d0444c4d 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go @@ -262,6 +262,27 @@ func TestCreateVoiceAgentAPIRequest_UsesAzureVoiceLocaleVariants(t *testing.T) { } } +func TestCreateVoiceAgentAPIRequest_PrefersExplicitVoiceLocale(t *testing.T) { + t.Parallel() + voiceLocale := "fr-FR" + agent := VoiceAgent{ + AgentDefinition: AgentDefinition{Kind: AgentKindPromptVoice, Name: "voice-agent"}, + Model: &Model{Id: "gpt-realtime"}, + Audio: &VoiceAudio{Output: &VoiceAudioOutput{Voice: &VoiceConfig{ + Type: "azure_standard", Name: "en-US-AvaNeural", Locale: &voiceLocale, + }}}, + } + + req, err := CreateVoiceAgentAPIRequest(agent) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + def := req.Definition.(agent_api.VoiceAgentDefinition) + if def.Audio.Output.VoiceLocale != voiceLocale { + t.Errorf("VoiceLocale = %q, want explicit %q", def.Audio.Output.VoiceLocale, voiceLocale) + } +} + func TestCreateVoiceAgentAPIRequest_MarshalServiceWireShape(t *testing.T) { t.Parallel() voice := "en-US-Ava:DragonHDLatestNeural" From cfdf03a929f873e6f128f974643f25cb727d7379 Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Wed, 26 Aug 2026 15:33:54 +0800 Subject: [PATCH 34/38] fix(agents): align hosted voice init with manifests --- .../azure.ai.agents/internal/cmd/init.go | 86 ++++++------- .../internal/cmd/init_from_code.go | 28 ++++- .../internal/cmd/init_hosted_voice.go | 110 ++++------------- .../internal/cmd/init_hosted_voice_test.go | 116 ++++++------------ 4 files changed, 121 insertions(+), 219 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/init.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/init.go index b9bb7b28c1c..6581a3193bd 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/init.go @@ -371,17 +371,6 @@ func resolveAgentNameFromManifestPointer( flags.agentName = validated return validated, nil } - if descriptor, compatible, err := loadHostedVoiceSourceDescriptor(manifestPointer); err != nil { - return "", fmt.Errorf("reading hosted voice sample descriptor: %w", err) - } else if compatible { - validated, err := validateInitAgentName(descriptor.Name) - if err != nil { - return "", err - } - flags.agentName = validated - return validated, nil - } - peeked := peekManifestName(ctx, manifestPointer, httpClient) if peeked == "" { // Defer to the inner flow which has access to the fully-loaded manifest. @@ -1081,38 +1070,6 @@ func runInitFromManifest( createdFolderDisplay string, userProvidedManifest bool, ) error { - if descriptor, compatible, err := loadHostedVoiceSourceDescriptor(flags.manifestPointer); err != nil { - return fmt.Errorf("reading hosted voice sample descriptor: %w", err) - } else if compatible { - cwd, err := os.Getwd() - if err != nil { - return fmt.Errorf("getting current directory for hosted voice sample: %w", err) - } - if !isSamePath(filepath.Dir(flags.manifestPointer), cwd) { - return exterrors.Validation( - exterrors.CodeInvalidParameter, - "hosted voice sample descriptor must be initialized from its source directory", - fmt.Sprintf("change directory to %q and run 'azd ai agent init -m %s'", filepath.Dir(flags.manifestPointer), filepath.Base(flags.manifestPointer)), - ) - } - if !promptVoicePreviewEnabled() { - return exterrors.Validation( - exterrors.CodeInvalidParameter, - "hosted voice agent init is private preview", - fmt.Sprintf("set %s=true to enable hosted voice init", promptVoicePreviewEnvVar), - ) - } - if err := applyHostedVoiceSourceDescriptor(flags, flags.manifestPointer, descriptor); err != nil { - return err - } - action := &InitFromCodeAction{ - azdClient: azdClient, - flags: flags, - httpClient: httpClient, - } - return action.Run(ctx) - } - // Ensure project and environment exist (no subscription/location prompting yet) projectConfig, err := ensureProject(ctx, flags, azdClient, targetDir) if err != nil { @@ -1585,7 +1542,7 @@ from code-deploy ZIP packaging (uses .gitignore syntax).`, if findErr != nil { return findErr } - if existing != "" && !isHostedVoiceSourceDescriptor(existing) { + if existing != "" { useExisting := flags.noPrompt if !flags.noPrompt { confirmResp, promptErr := azdClient.Prompt().Confirm(ctx, &azdext.ConfirmRequest{ @@ -2074,6 +2031,14 @@ func (a *InitAction) Run(ctx context.Context) error { if err != nil { return fmt.Errorf("downloading agent.yaml: %w", err) } + if _, hostedVoice, err := hostedVoiceManifestTarget(agentManifest); err != nil { + return err + } else if hostedVoice { + a.flags.kind = kindFlagHostedVoice + if err := validateHostedVoiceServiceNamesForProject(ctx, a.azdClient, a.serviceNameOverride); err != nil { + return err + } + } // Prompt for deploy mode (code vs container) for hosted agents. // Code deploy is supported for Python and .NET projects. @@ -2087,10 +2052,27 @@ func (a *InitAction) Run(ctx context.Context) error { if a.isCodeDeploy { // Prompt for code configuration and update the manifest - codeConfig, err := promptCodeConfig(ctx, a.azdClient, targetDir, a.flags.noPrompt, codeDeployOptions{ + codeOptions := codeDeployOptions{ runtime: a.flags.runtime, entryPoint: a.flags.entryPoint, depResolution: a.flags.depResolution, + } + if hostedAgent, ok := agentManifest.Template.(agent_yaml.ContainerAgent); ok && + hostedAgent.CodeConfiguration != nil { + if codeOptions.runtime == "" { + codeOptions.runtime = hostedAgent.CodeConfiguration.Runtime + } + if codeOptions.entryPoint == "" { + codeOptions.entryPoint = hostedAgent.CodeConfiguration.EntryPoint + } + if codeOptions.depResolution == "" && hostedAgent.CodeConfiguration.DependencyResolution != nil { + codeOptions.depResolution = *hostedAgent.CodeConfiguration.DependencyResolution + } + } + codeConfig, err := promptCodeConfig(ctx, a.azdClient, targetDir, a.flags.noPrompt, codeDeployOptions{ + runtime: codeOptions.runtime, + entryPoint: codeOptions.entryPoint, + depResolution: codeOptions.depResolution, }, a.userProvidedManifest) if err != nil { return fmt.Errorf("prompting for code configuration: %w", err) @@ -2185,6 +2167,11 @@ func (a *InitAction) Run(ctx context.Context) error { if err := a.addToProject(ctx, targetDir, agentManifest); err != nil { return fmt.Errorf("failed to add agent to azure.yaml: %w", err) } + if strings.EqualFold(a.flags.kind, kindFlagHostedVoice) { + if err := addHostedVoiceWrapperToProject(ctx, a.azdClient, a.serviceNameOverride); err != nil { + return fmt.Errorf("adding hosted voice wrapper: %w", err) + } + } // Run post-init validations (advisory warnings only) if ca, ok := agentManifest.Template.(agent_yaml.ContainerAgent); ok { @@ -4510,8 +4497,15 @@ func (a *InitAction) validateCodeDeployFlags() error { if err := validateImageFlag(a.flags.image, a.flags.deployMode); err != nil { return err } + noPrompt := a.flags.noPrompt + if a.flags.manifestPointer != "" { + // A standard manifest can provide runtime, entry point, and dependency + // resolution. Validate values now and enforce completeness after the + // manifest has been loaded and merged with explicit CLI overrides. + noPrompt = false + } return validateCodeDeployInput( - a.flags.noPrompt, a.flags.deployMode, a.flags.runtime, a.flags.entryPoint, a.flags.depResolution) + noPrompt, a.flags.deployMode, a.flags.runtime, a.flags.entryPoint, a.flags.depResolution) } var initImageRefRe = regexp.MustCompile( diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_code.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_code.go index e6e56cbd088..b3c9ff021e5 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_code.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_code.go @@ -85,7 +85,7 @@ func (a *InitFromCodeAction) Run(ctx context.Context) error { // when the user declined the reuse prompt in RunE or bypassed it; we still // refuse in --no-prompt and confirm interactively. if existing, statErr := findExistingAgentYaml(srcDir); statErr == nil && existing != "" && - !isHostedVoiceSourceDescriptor(existing) && !strings.EqualFold(a.flags.kind, kindFlagHostedVoice) { + !strings.EqualFold(a.flags.kind, kindFlagHostedVoice) { displayPath, relErr := filepath.Rel(srcDir, existing) if relErr != nil || displayPath == "" { displayPath = existing @@ -945,7 +945,15 @@ func (a *InitFromCodeAction) addToProject( } func (a *InitFromCodeAction) validateHostedVoiceServiceNames(ctx context.Context, targetServiceName string) error { - response, err := a.azdClient.Project().Get(ctx, &azdext.EmptyRequest{}) + return validateHostedVoiceServiceNamesForProject(ctx, a.azdClient, targetServiceName) +} + +func validateHostedVoiceServiceNamesForProject( + ctx context.Context, + azdClient *azdext.AzdClient, + targetServiceName string, +) error { + response, err := azdClient.Project().Get(ctx, &azdext.EmptyRequest{}) if err != nil { return fmt.Errorf("checking existing services for hosted voice init: %w", err) } @@ -965,8 +973,16 @@ func (a *InitFromCodeAction) validateHostedVoiceServiceNames(ctx context.Context } func (a *InitFromCodeAction) addHostedVoiceWrapper(ctx context.Context, targetServiceName string) error { + return addHostedVoiceWrapperToProject(ctx, a.azdClient, targetServiceName) +} + +func addHostedVoiceWrapperToProject( + ctx context.Context, + azdClient *azdext.AzdClient, + targetServiceName string, +) error { wrapperName := hostedVoiceWrapperName(targetServiceName) - response, err := a.azdClient.Project().Get(ctx, &azdext.EmptyRequest{}) + response, err := azdClient.Project().Get(ctx, &azdext.EmptyRequest{}) if err != nil || response.Project == nil { return fmt.Errorf("loading project services before adding hosted voice wrapper: %w", err) } @@ -977,7 +993,7 @@ func (a *InitFromCodeAction) addHostedVoiceWrapper(ctx context.Context, targetSe "choose a different target agent name so the generated wrapper service name is unique", ) } - projectServiceName := existingProjectServiceKey(ctx, a.azdClient) + projectServiceName := existingProjectServiceKey(ctx, azdClient) if projectServiceName == "" { return fmt.Errorf("cannot resolve the azure.ai.project service for hosted voice wrapper %q", wrapperName) } @@ -1000,7 +1016,7 @@ func (a *InitFromCodeAction) addHostedVoiceWrapper(ctx context.Context, targetSe if err != nil { return err } - if _, err := a.azdClient.Project().AddService(ctx, &azdext.AddServiceRequest{Service: &azdext.ServiceConfig{ + if _, err := azdClient.Project().AddService(ctx, &azdext.AddServiceRequest{Service: &azdext.ServiceConfig{ Name: wrapperName, Host: AiAgentHost, AdditionalProperties: props, @@ -1008,7 +1024,7 @@ func (a *InitFromCodeAction) addHostedVoiceWrapper(ctx context.Context, targetSe return fmt.Errorf("adding hosted voice wrapper service: %w", err) } - if err := setServiceUses(ctx, a.azdClient, wrapperName, []string{projectServiceName, targetServiceName}); err != nil { + if err := setServiceUses(ctx, azdClient, wrapperName, []string{projectServiceName, targetServiceName}); err != nil { return err } fmt.Printf(" %s Added hosted voice wrapper %s -> %s\n", color.GreenString("+"), wrapperName, targetServiceName) diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_hosted_voice.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_hosted_voice.go index 8999434d1ae..1f49781dfe3 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_hosted_voice.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_hosted_voice.go @@ -4,108 +4,44 @@ package cmd import ( - "azureaiagent/internal/exterrors" - "fmt" - "os" - "path/filepath" - "slices" "strings" + "azureaiagent/internal/pkg/agents/agent_yaml" + "gopkg.in/yaml.v3" ) -// hostedVoiceSourceDescriptor is source capability metadata used by the Voice -// Bridge samples. It is not an azd agent definition and must not be adopted or -// overwritten by init. -type hostedVoiceSourceDescriptor struct { - Name string `yaml:"name"` - Protocols []string `yaml:"protocols"` - VoiceLiveCompatible string `yaml:"voiceLiveCompatible"` - BridgeProtocolVersion string `yaml:"bridgeProtocolVersion"` -} - -func loadHostedVoiceSourceDescriptor(path string) (*hostedVoiceSourceDescriptor, bool, error) { - if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") { +func hostedVoiceManifestTarget(manifest *agent_yaml.AgentManifest) (*agent_yaml.ContainerAgent, bool, error) { + if manifest == nil { return nil, false, nil } - content, err := os.ReadFile(path) //nolint:gosec // path was discovered under the selected source directory + templateYAML, err := yaml.Marshal(manifest.Template) if err != nil { - if os.IsNotExist(err) { - return nil, false, nil - } - return nil, false, err + return nil, false, fmt.Errorf("marshaling hosted voice manifest template: %w", err) } - var raw map[string]any - if err := yaml.Unmarshal(content, &raw); err != nil { + var target agent_yaml.ContainerAgent + if err := yaml.Unmarshal(templateYAML, &target); err != nil { return nil, false, nil } - if _, isAzdManifest := raw["template"]; isAzdManifest { + if target.Kind != agent_yaml.AgentKindHosted { return nil, false, nil } - var descriptor hostedVoiceSourceDescriptor - if err := yaml.Unmarshal(content, &descriptor); err != nil { - return nil, false, nil - } - protocols := make([]string, 0, len(descriptor.Protocols)) - for _, protocol := range descriptor.Protocols { - protocols = append(protocols, strings.ToLower(strings.TrimSpace(protocol))) - } - compatible := strings.TrimSpace(descriptor.Name) != "" && - slices.Contains(protocols, "invocations_ws") && - strings.EqualFold(strings.TrimSpace(descriptor.VoiceLiveCompatible), "true") && - strings.TrimSpace(descriptor.BridgeProtocolVersion) == "1.0" - return &descriptor, compatible, nil -} - -func isHostedVoiceSourceDescriptor(path string) bool { - _, compatible, err := loadHostedVoiceSourceDescriptor(path) - return err == nil && compatible -} - -func applyHostedVoiceSourceDescriptor(flags *initFlags, path string, descriptor *hostedVoiceSourceDescriptor) error { - if flags == nil || descriptor == nil { - return fmt.Errorf("hosted voice source descriptor is required") - } - name, err := validateInitAgentName(descriptor.Name) - if err != nil { - return err - } - sourceDir, err := filepath.Abs(filepath.Dir(path)) - if err != nil { - return fmt.Errorf("resolving hosted voice sample directory: %w", err) - } - flags.kind = kindFlagHostedVoice - if strings.TrimSpace(flags.agentName) == "" { - flags.agentName = name - } - flags.src = sourceDir - flags.deployMode = "code" - flags.protocols = []string{"invocations_ws"} - flags.depResolution = "bundled" - if isDotnetProject(sourceDir) { - flags.runtime = "dotnet_10" - } else if isPythonProject(sourceDir) { - flags.runtime = "python_3_13" - } - if flags.runtime == "" { - return exterrors.Validation( - exterrors.CodeInvalidParameter, - "could not detect a supported runtime for the hosted voice sample", - "place agent.manifest.yaml next to a .NET or Python agent project", - ) + compatibleProtocol := false + for _, protocol := range target.Protocols { + if protocol.Protocol == "invocations_ws" && protocol.Version == "1.0.0" { + compatibleProtocol = true + break + } } - flags.entryPoint = detectDefaultEntryPoint(sourceDir, flags.runtime) - validEntryPoint := strings.TrimSpace(flags.entryPoint) != "" - if flags.runtime != "dotnet_10" { - validEntryPoint = validEntryPoint && fileExists(filepath.Join(sourceDir, flags.entryPoint)) + if !compatibleProtocol || target.Metadata == nil { + return nil, false, nil } - if !validEntryPoint { - return exterrors.Validation( - exterrors.CodeInvalidParameter, - "could not detect the hosted voice sample entry point", - "ensure the sample contains its expected .NET assembly or Python entry module", - ) + metadata := *target.Metadata + voiceCompatible := strings.EqualFold(strings.TrimSpace(fmt.Sprint(metadata["voiceLiveCompatible"])), "true") + bridgeVersion := strings.TrimSpace(fmt.Sprint(metadata["bridgeProtocolVersion"])) + if !voiceCompatible || bridgeVersion != "1.0" { + return nil, false, nil } - return nil + return &target, true, nil } diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_hosted_voice_test.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_hosted_voice_test.go index 9aa4989246c..0d9135e969f 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_hosted_voice_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_hosted_voice_test.go @@ -4,99 +4,55 @@ package cmd import ( - "os" - "path/filepath" "testing" + "azureaiagent/internal/pkg/agents/agent_yaml" + "github.com/stretchr/testify/require" ) -func TestIsHostedVoiceSourceDescriptor(t *testing.T) { - t.Parallel() - path := filepath.Join(t.TempDir(), "agent.manifest.yaml") - require.NoError(t, os.WriteFile(path, []byte(` -name: voice-hosted-agent-dotnet -protocols: - - invocations_ws -voiceLiveCompatible: "true" -bridgeProtocolVersion: "1.0" -`), 0600)) - require.True(t, isHostedVoiceSourceDescriptor(path)) -} - -func TestIsHostedVoiceSourceDescriptorRejectsIncompatibleDescriptor(t *testing.T) { - t.Parallel() - path := filepath.Join(t.TempDir(), "agent.manifest.yaml") - require.NoError(t, os.WriteFile(path, []byte(` -protocols: - - responses -voiceLiveCompatible: "true" -bridgeProtocolVersion: "2.0" -`), 0600)) - require.False(t, isHostedVoiceSourceDescriptor(path)) -} - -func TestApplyHostedVoiceSourceDescriptorDotnet(t *testing.T) { +func TestHostedVoiceManifestTarget(t *testing.T) { t.Parallel() - dir := t.TempDir() - manifestPath := filepath.Join(dir, "agent.manifest.yaml") - require.NoError(t, os.WriteFile(manifestPath, []byte("name: voice-sample\n"), 0600)) - require.NoError(t, os.WriteFile(filepath.Join(dir, "VoiceHostedAgent.csproj"), []byte(` -VoiceHostedAgent -`), 0600)) - flags := &initFlags{} - err := applyHostedVoiceSourceDescriptor(flags, manifestPath, &hostedVoiceSourceDescriptor{Name: "voice-sample"}) + metadata := map[string]any{ + "voiceLiveCompatible": "true", + "bridgeProtocolVersion": "1.0", + } + manifest := &agent_yaml.AgentManifest{Template: agent_yaml.ContainerAgent{ + AgentDefinition: agent_yaml.AgentDefinition{ + Kind: agent_yaml.AgentKindHosted, Name: "voice-target", Metadata: &metadata, + }, + Protocols: []agent_yaml.ProtocolVersionRecord{{Protocol: "invocations_ws", Version: "1.0.0"}}, + }} + target, compatible, err := hostedVoiceManifestTarget(manifest) require.NoError(t, err) - require.Equal(t, kindFlagHostedVoice, flags.kind) - require.Equal(t, "voice-sample", flags.agentName) - require.Equal(t, dir, flags.src) - require.Equal(t, "code", flags.deployMode) - require.Equal(t, "dotnet_10", flags.runtime) - require.Equal(t, "VoiceHostedAgent.dll", flags.entryPoint) - require.Equal(t, "bundled", flags.depResolution) - require.Equal(t, []string{"invocations_ws"}, flags.protocols) -} - -func TestApplyHostedVoiceSourceDescriptorPreservesExplicitName(t *testing.T) { - t.Parallel() - dir := t.TempDir() - manifestPath := filepath.Join(dir, "agent.manifest.yaml") - require.NoError(t, os.WriteFile(manifestPath, []byte("name: descriptor-name\n"), 0600)) - require.NoError(t, os.WriteFile(filepath.Join(dir, "VoiceHostedAgent.csproj"), []byte(` -VoiceHostedAgent -`), 0600)) - flags := &initFlags{agentName: "explicit-name"} - require.NoError(t, applyHostedVoiceSourceDescriptor( - flags, manifestPath, &hostedVoiceSourceDescriptor{Name: "descriptor-name"}, - )) - require.Equal(t, "explicit-name", flags.agentName) + require.True(t, compatible) + require.Equal(t, "voice-target", target.Name) } -func TestApplyHostedVoiceSourceDescriptorRejectsMissingEntrypoint(t *testing.T) { +func TestHostedVoiceManifestTargetRejectsIncompatibleManifest(t *testing.T) { t.Parallel() - dir := t.TempDir() - manifestPath := filepath.Join(dir, "agent.manifest.yaml") - require.NoError(t, os.WriteFile(manifestPath, []byte("name: voice-sample\n"), 0600)) - require.NoError(t, os.WriteFile(filepath.Join(dir, "requirements.txt"), []byte("websockets\n"), 0600)) - err := applyHostedVoiceSourceDescriptor( - &initFlags{}, manifestPath, &hostedVoiceSourceDescriptor{Name: "voice-sample"}, - ) - require.ErrorContains(t, err, "could not detect the hosted voice sample entry point") + metadata := map[string]any{ + "voiceLiveCompatible": "true", + "bridgeProtocolVersion": "2.0", + } + manifest := &agent_yaml.AgentManifest{Template: agent_yaml.ContainerAgent{ + AgentDefinition: agent_yaml.AgentDefinition{ + Kind: agent_yaml.AgentKindHosted, Name: "voice-target", Metadata: &metadata, + }, + Protocols: []agent_yaml.ProtocolVersionRecord{{Protocol: "invocations_ws", Version: "1.0.0"}}, + }} + _, compatible, err := hostedVoiceManifestTarget(manifest) + require.NoError(t, err) + require.False(t, compatible) } -func TestHostedVoiceDescriptorDoesNotMatchAzdManifest(t *testing.T) { +func TestHostedVoiceManifestTargetRejectsGenericInvocationsWS(t *testing.T) { t.Parallel() - path := filepath.Join(t.TempDir(), "agent.yaml") - require.NoError(t, os.WriteFile(path, []byte(` -name: manifest -protocols: [invocations_ws] -voiceLiveCompatible: "true" -bridgeProtocolVersion: "1.0" -template: - kind: hosted - name: agent -`), 0600)) - _, compatible, err := loadHostedVoiceSourceDescriptor(path) + manifest := &agent_yaml.AgentManifest{Template: agent_yaml.ContainerAgent{ + AgentDefinition: agent_yaml.AgentDefinition{Kind: agent_yaml.AgentKindHosted, Name: "generic"}, + Protocols: []agent_yaml.ProtocolVersionRecord{{Protocol: "invocations_ws", Version: "1.0.0"}}, + }} + _, compatible, err := hostedVoiceManifestTarget(manifest) require.NoError(t, err) require.False(t, compatible) } From 41f126cc37cd07226ae8972e0968bd64365cffa7 Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Wed, 26 Aug 2026 15:56:01 +0800 Subject: [PATCH 35/38] docs(agents): document hosted voice manifest flow --- cli/azd/extensions/azure.ai.agents/README.md | 17 +++++++++++------ .../docs/hosted-voice-test-guide.md | 11 +++++------ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/README.md b/cli/azd/extensions/azure.ai.agents/README.md index c93edc97c15..99c7848f29f 100644 --- a/cli/azd/extensions/azure.ai.agents/README.md +++ b/cli/azd/extensions/azure.ai.agents/README.md @@ -253,10 +253,16 @@ keys throughout this block (`invocations_moderation`, `response_mode`, A hosted voice wrapper keeps Voice Live responsible for VAD, speech-to-text, and text-to-speech while routing conversation logic to a hosted agent in the -same Foundry project. Run interactive init from compatible Voice Bridge source -code by setting `AZD_AI_AGENT_ENABLE_PROMPT_VOICE=true` and selecting **Create a -hosted voice agent from the code in the current directory**. For CI, use -`azd ai agent init --kind hosted-voice ... --no-prompt`. +same Foundry project. Hosted Voice samples use the same standard Agent Manifest +flow as other Hosted Agent and `invocations_ws` samples: + +```powershell +azd ai agent init -m +``` + +When the sample source is already present, run `azd ai agent init` from its +directory and accept the detected local manifest. azd reuses a parent project +when the source is already inside an existing azd project. Init generates both services and references the target by its `azure.yaml` service name: @@ -300,8 +306,7 @@ services: The `uses` edge deploys the target before the wrapper. `version: deployed` pins the wrapper to the target version produced by the current azd environment. -Hosted voice wrappers automatically use the unified flat Voice API unless -`AZURE_VOICE_AGENT_API` is explicitly set to an incompatible mode. +Hosted voice wrappers use the unified Voice API. The target must be active, declare `invocations_ws/1.0.0`, and include `voiceLiveCompatible=true` and `bridgeProtocolVersion=1.0` metadata. Model, diff --git a/cli/azd/extensions/azure.ai.agents/docs/hosted-voice-test-guide.md b/cli/azd/extensions/azure.ai.agents/docs/hosted-voice-test-guide.md index a09c8c4b977..399bfcb7254 100644 --- a/cli/azd/extensions/azure.ai.agents/docs/hosted-voice-test-guide.md +++ b/cli/azd/extensions/azure.ai.agents/docs/hosted-voice-test-guide.md @@ -42,16 +42,15 @@ same Foundry project. ## Manifest -Generate the composition interactively from compatible source code: +Initialize an empty folder from the standard public Agent Manifest: ```powershell -$env:AZD_AI_AGENT_ENABLE_PROMPT_VOICE = "true" -azd ai agent init +azd ai agent init -m ``` -Select **Create a hosted voice agent from the code in the current directory**. -For CI, use `--kind hosted-voice` with the normal code deploy flags and -`--no-prompt`. +When source already exists, run `azd ai agent init` without `-m` from the source +directory. azd detects the local manifest and reuses a parent azd project when +present. Use one project service, one hosted target, and one Voice wrapper. The wrapper references the target by its `azure.yaml` service name, not by a remote agent From 0628ae4babab28956ee43df2b837bf73bec8c74c Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:01:48 +0800 Subject: [PATCH 36/38] fix(agents): preserve g711 voice rates --- .../internal/pkg/agents/agent_yaml/map.go | 2 ++ .../pkg/agents/agent_yaml/map_voice_test.go | 24 +++++++++++++++++++ .../schemas/azure.ai.agent.json | 1 + 3 files changed, 27 insertions(+) diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go index 0e2dce82e0d..1e2ed16b3df 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go @@ -626,6 +626,8 @@ func mapVoiceAudioFormat(format *VoiceAudioFormat, fallback *agent_api.VoiceAudi } if format.Rate != nil { out.Rate = format.Rate + } else if out.Type == "audio/pcmu" || out.Type == "audio/pcma" { + out.Rate = nil } } return out diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go index a72d0444c4d..46f2847f419 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go @@ -283,6 +283,30 @@ func TestCreateVoiceAgentAPIRequest_PrefersExplicitVoiceLocale(t *testing.T) { } } +func TestCreateVoiceAgentAPIRequest_DoesNotInheritPcmRateForG711Formats(t *testing.T) { + t.Parallel() + agent := VoiceAgent{ + AgentDefinition: AgentDefinition{Kind: AgentKindPromptVoice, Name: "voice-agent"}, + Model: &Model{Id: "gpt-realtime"}, + Audio: &VoiceAudio{ + Input: &VoiceAudioInput{Format: &VoiceAudioFormat{Type: "audio/pcmu"}}, + Output: &VoiceAudioOutput{Format: &VoiceAudioFormat{Type: "audio/pcma"}}, + }, + } + + req, err := CreateVoiceAgentAPIRequest(agent) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + def := req.Definition.(agent_api.VoiceAgentDefinition) + if def.Audio.Input.Format.Rate != nil { + t.Fatalf("input G.711 rate = %v, want nil", *def.Audio.Input.Format.Rate) + } + if def.Audio.Output.Format.Rate != nil { + t.Fatalf("output G.711 rate = %v, want nil", *def.Audio.Output.Format.Rate) + } +} + func TestCreateVoiceAgentAPIRequest_MarshalServiceWireShape(t *testing.T) { t.Parallel() voice := "en-US-Ava:DragonHDLatestNeural" diff --git a/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json b/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json index c5319e87ec5..a500d46d8b7 100644 --- a/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json +++ b/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json @@ -116,6 +116,7 @@ "description": "Voice agent (kind: prompt-voice) parallel tool call toggle. Currently rejected by azd for prompt voice runtime." }, "maxOutputTokens": { + "type": ["integer", "string"], "description": "Voice agent (kind: prompt-voice) maximum output tokens. Use an integer or a service-supported string such as inf." }, "include": { From 44233cbf29501283044416bb180dc6be4c6a933d Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:12:01 +0800 Subject: [PATCH 37/38] test(agents): cover voice advanced validation --- .../internal/pkg/agents/agent_yaml/map.go | 6 ++ .../pkg/agents/agent_yaml/map_voice_test.go | 53 ++++++++++--- .../pkg/agents/agent_yaml/parse_voice_test.go | 78 +++++++++++++++++++ 3 files changed, 127 insertions(+), 10 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go index 1e2ed16b3df..0a870ef903e 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go @@ -685,6 +685,9 @@ func mapVoiceConfig(voice *VoiceConfig, fallbackName string) *agent_api.VoiceCon out.Volume = voice.Volume return out } + if voiceType == "openai" { + name = strings.ToLower(name) + } return &agent_api.VoiceConfig{ Type: voiceType, Name: name, @@ -745,6 +748,9 @@ func createVoiceAgentAPIRequest(voiceAgent VoiceAgent) (*agent_api.CreateAgentRe "model_type '%s' is not supported; use '%s' or '%s'", voiceAgent.ModelType, VoiceModelTypeManaged, VoiceModelTypeSelfDeployed) } + if errors := validateVoiceAgentAdvancedConfig(voiceAgent); len(errors) > 0 { + return nil, fmt.Errorf("invalid prompt-voice configuration: %s", strings.Join(errors, "; ")) + } instructions := defaultVoiceInstructions if voiceAgent.Instructions != nil && *voiceAgent.Instructions != "" { diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go index 46f2847f419..166d1b60eb0 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go @@ -5,6 +5,7 @@ package agent_yaml import ( "encoding/json" + "strings" "testing" "azureaiagent/internal/pkg/agents/agent_api" @@ -283,6 +284,40 @@ func TestCreateVoiceAgentAPIRequest_PrefersExplicitVoiceLocale(t *testing.T) { } } +func TestCreateVoiceAgentAPIRequest_ExplicitOpenAIVoiceLowercasesName(t *testing.T) { + t.Parallel() + agent := VoiceAgent{ + AgentDefinition: AgentDefinition{Kind: AgentKindPromptVoice, Name: "voice-agent"}, + Model: &Model{Id: "gpt-realtime"}, + Audio: &VoiceAudio{Output: &VoiceAudioOutput{Voice: &VoiceConfig{ + Type: "openai", Name: "Shimmer", + }}}, + } + + req, err := CreateVoiceAgentAPIRequest(agent) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + def := req.Definition.(agent_api.VoiceAgentDefinition) + if def.Audio.Output.Voice != "shimmer" { + t.Errorf("Voice = %q, want shimmer", def.Audio.Output.Voice) + } +} + +func TestCreateVoiceAgentAPIRequest_RejectsInvalidAdvancedConfig(t *testing.T) { + t.Parallel() + parallelToolCalls := true + agent := VoiceAgent{ + AgentDefinition: AgentDefinition{Kind: AgentKindPromptVoice, Name: "voice-agent"}, + Model: &Model{Id: "gpt-realtime"}, + ParallelToolCalls: ¶llelToolCalls, + } + _, err := CreateVoiceAgentAPIRequest(agent) + if err == nil || !strings.Contains(err.Error(), "parallel_tool_calls is not currently supported") { + t.Fatalf("expected parallel_tool_calls validation error, got: %v", err) + } +} + func TestCreateVoiceAgentAPIRequest_DoesNotInheritPcmRateForG711Formats(t *testing.T) { t.Parallel() agent := VoiceAgent{ @@ -367,7 +402,6 @@ func TestCreateVoiceAgentAPIRequest_AdvancedSettingsWireShape(t *testing.T) { interruptResponse := true autoTruncate := true speed := 1.1 - parallelToolCalls := true style := "cheerful" pitch := "+0Hz" rate := "+0%" @@ -394,7 +428,7 @@ func TestCreateVoiceAgentAPIRequest_AdvancedSettingsWireShape(t *testing.T) { Languages: []string{"en-US"}, AutoTruncate: &autoTruncate, }, - Transcription: &VoiceTranscription{Model: "whisper-1", Language: new("en-US"), Prompt: new("Contoso terms")}, + Transcription: &VoiceTranscription{Model: "azure-speech", Language: new("en-US"), Prompt: new("Contoso terms")}, }, Output: &VoiceAudioOutput{ Format: &VoiceAudioFormat{Type: "audio/pcm", Rate: &outRate}, @@ -405,14 +439,13 @@ func TestCreateVoiceAgentAPIRequest_AdvancedSettingsWireShape(t *testing.T) { Speed: &speed, }, }, - OutputModalities: []string{"audio", "text"}, - Tools: []map[string]any{{"type": "system", "name": "end_conversation"}}, - Avatar: map[string]any{"type": "video_avatar", "character": "lisa", "output_protocol": "webrtc"}, - Greeting: map[string]any{"type": "template", "text": "Hello {{persona}}"}, - ToolChoice: "auto", - ParallelToolCalls: ¶llelToolCalls, - MaxOutputTokens: "inf", - Include: []string{"item.input_audio_transcription.phrases"}, + OutputModalities: []string{"audio", "text"}, + Tools: []map[string]any{{"type": "system", "name": "end_conversation"}}, + Avatar: map[string]any{"type": "video_avatar", "character": "lisa", "output_protocol": "webrtc"}, + Greeting: map[string]any{"type": "template", "text": "Hello {{persona}}"}, + ToolChoice: "auto", + MaxOutputTokens: "inf", + Include: []string{"item.input_audio_transcription.phrases"}, } req, err := CreateVoiceAgentAPIRequest(agent) diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse_voice_test.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse_voice_test.go index 577754832fd..c3ffb44ee37 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse_voice_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse_voice_test.go @@ -172,3 +172,81 @@ include: t.Fatalf("expected transcription model guidance in error, got: %v", err) } } + +func TestValidateAgentDefinition_PromptVoice_AdvancedValidationBoundaries(t *testing.T) { + tests := []struct { + name string + yaml string + want string + }{ + { + name: "unsupported format", + yaml: `audio: + input: + format: + type: audio/opus`, + want: "audio/pcm", + }, + { + name: "invalid rate", + yaml: `audio: + input: + format: + type: audio/pcm + rate: 0`, + want: "rate must be greater than 0", + }, + { + name: "negative duration", + yaml: `audio: + input: + turn_detection: + type: azure_semantic_vad + speech_duration_ms: -1`, + want: "speech_duration_ms must be >= 0", + }, + { + name: "blank voice name", + yaml: `audio: + output: + voice: + type: azure_standard + name: ""`, + want: "voice.name must not be blank", + }, + { + name: "invalid speed", + yaml: `audio: + output: + speed: 2`, + want: "speed must be between 0.25 and 1.5", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + yamlContent := []byte("kind: prompt-voice\nname: voice-agent\nmodel:\n id: gpt-realtime\n" + tt.yaml + "\n") + err := ValidateAgentDefinition(yamlContent) + if err == nil || !strings.Contains(err.Error(), tt.want) { + t.Fatalf("expected %q validation error, got: %v", tt.want, err) + } + }) + } +} + +func TestValidateAgentDefinition_PromptVoice_ValidIncludeTranscriptionModel(t *testing.T) { + yamlContent := []byte(` +kind: prompt-voice +name: voice-agent +model: + id: gpt-realtime +audio: + input: + transcription: + model: azure-speech +include: + - item.input_audio_transcription.phrases +`) + if err := ValidateAgentDefinition(yamlContent); err != nil { + t.Fatalf("expected azure-speech include config to be valid, got: %v", err) + } +} From ab3adc175a70ffe84274a7f9bd2618842ce8d934 Mon Sep 17 00:00:00 2001 From: Jian Wu <223556219+Copilot@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:22:30 +0800 Subject: [PATCH 38/38] test(agents): cover manifest code defaults --- .../extensions/azure.ai.agents/internal/cmd/init_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_test.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_test.go index 083ac34a231..441a8114820 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_test.go @@ -3039,6 +3039,13 @@ func TestCodeDeployFlagValidation(t *testing.T) { flags: initFlags{noPrompt: false, deployMode: "code"}, wantErr: false, }, + { + name: "no-prompt manifest can provide code configuration", + flags: initFlags{ + noPrompt: true, deployMode: "code", manifestPointer: "agent.manifest.yaml", + }, + wantErr: false, + }, { name: "invalid deploy-mode value fails", flags: initFlags{noPrompt: true, deployMode: "invalid"},