diff --git a/cli/azd/extensions/azure.ai.agents/README.md b/cli/azd/extensions/azure.ai.agents/README.md index f6c5760caf0..28c95c99e22 100644 --- a/cli/azd/extensions/azure.ai.agents/README.md +++ b/cli/azd/extensions/azure.ai.agents/README.md @@ -263,6 +263,164 @@ keys throughout this block (`invocations_moderation`, `response_mode`, `input_paths`, `stream_selectors`, `event_type`, and so on). The **values** (`non_streaming`, `streaming`, `both`, `json`, `text`) are the same in both. +### Hosted voice wrapper (preview) + +A hosted voice wrapper keeps Voice Live responsible for VAD, speech-to-text, +and text-to-speech while routing conversation logic to a hosted agent in the +same Foundry project. Hosted Voice samples use the same standard Agent Manifest +flow as other Hosted Agent and `invocations_ws` samples: + +```powershell +azd ai agent init -m +``` + +When the sample source is already present, run `azd ai agent init` from its +directory and accept the detected local manifest. azd reuses a parent project +when the source is already inside an existing azd project. + +Init generates both services and references the target by its `azure.yaml` +service name: + +```yaml +services: + voice-target: + host: azure.ai.agent + project: ./src/voice-target + language: csharp + kind: hosted + name: voice-target + protocols: + - protocol: invocations_ws + version: 1.0.0 + metadata: + voiceLiveCompatible: "true" + bridgeProtocolVersion: "1.0" + codeConfiguration: + runtime: dotnet_10 + entryPoint: VoiceHostedAgent.dll + dependencyResolution: bundled + + voice: + host: azure.ai.agent + kind: prompt-voice + name: voice + uses: + - voice-target + modelType: hosted_agent + targetAgent: + service: voice-target + version: deployed + store: false + audio: + output: + voice: + type: azure_standard + name: en-US-JennyNeural +``` + +The `uses` edge deploys the target before the wrapper. `version: deployed` +pins the wrapper to the target version produced by the current azd environment. +Hosted voice wrappers use the unified Voice API. + +The target must be active, declare `invocations_ws/1.0.0`, and include +`voiceLiveCompatible=true` and `bridgeProtocolVersion=1.0` metadata. Model, +instructions, tools, and other conversation controls belong to the target; +the wrapper owns audio, voice, store, avatar, and greeting configuration. + +For end-to-end validation, lifecycle checks, local dashboard steps, and the +experience alignment matrix, see [Hosted Voice Agent Test Guide](docs/hosted-voice-test-guide.md). + +## Prompt voice advanced configuration + +Advanced prompt voice settings are authored on the `azure.ai.agent` service in +`azure.yaml` and require the unified flat API mode: + +```bash +azd env set AZURE_VOICE_AGENT_API unified-flat +``` + +```yaml +services: + voice-agent: + host: azure.ai.agent + kind: prompt-voice + name: voice-agent + modelType: managed # or self_deployed for BYOM + model: + id: gpt-realtime + instructions: You are {{persona}}, a concise voice assistant. + structuredInputs: + persona: + description: Assistant persona + defaultValue: Ada + schema: + type: string + audio: + input: + format: + type: audio/pcmu + noiseReduction: + type: near_field + echoCancellation: + type: server_echo_cancellation + reference_source: server + channels: 1 + turnDetection: + type: azure_semantic_vad + threshold: 0.6 + speechDurationMs: 120 + removeFillerWords: true + createResponse: true + interruptResponse: true + languages: [en-US] + autoTruncate: true + transcription: + model: azure-speech + language: en-US + output: + format: + type: audio/pcm + rate: 24000 + voice: + type: azure_standard + name: en-US-AvaNeural + locale: en-US + style: cheerful + speed: 1.0 + outputModalities: [audio, text] + tools: + - type: system + name: end_conversation + avatar: + type: video_avatar + character: lisa + style: casual-sitting + output_protocol: webrtc + greeting: + type: template + text: Hello {{persona}} + toolChoice: auto + maxOutputTokens: inf + include: + - item.input_audio_transcription.phrases +``` + +Notes: + +- `voice`, `instructions`, and `store` remain supported for simple prompt voice + agents. Missing audio fields keep the existing azd defaults. +- `audio.output.voice` uses an author-friendly object shape; in `unified-flat` + mode azd maps it to the service flat fields `voice`, `voice_type`, + `voice_locale`, `style`, `pitch`, `rate`, and `volume`. +- `structuredInputs.defaultValue` maps to the service wire field + `default_value`. +- Direct voice tool types are `function`, `mcp`, `system`, and `toolbox`. + Server-side tools such as `web_search`, `azure_ai_search`, and `openapi` must + be packaged through a toolbox. +- `tools`, `avatar`, `greeting`, `handoff`, `toolChoice`, and + `echoCancellation` intentionally remain light pass-through blocks so azd does + not block new service-side additions. + ## Session idle timeout A hosted agent's runtime session sandbox is suspended by Foundry after a period diff --git a/cli/azd/extensions/azure.ai.agents/cspell.yaml b/cli/azd/extensions/azure.ai.agents/cspell.yaml index 5d2d9e0950c..78be91d2fea 100644 --- a/cli/azd/extensions/azure.ai.agents/cspell.yaml +++ b/cli/azd/extensions/azure.ai.agents/cspell.yaml @@ -11,6 +11,9 @@ words: # Voice (prompt-voice) agents - BYOM - Nanami + - pcma + - pcmu + - webrtc # Azure region names - australiaeast - brazilsouth diff --git a/cli/azd/extensions/azure.ai.agents/docs/hosted-voice-test-guide.md b/cli/azd/extensions/azure.ai.agents/docs/hosted-voice-test-guide.md new file mode 100644 index 00000000000..399bfcb7254 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/docs/hosted-voice-test-guide.md @@ -0,0 +1,334 @@ +# Hosted Voice Agent Test Guide + +This guide validates the preview `azd` Hosted Voice Agent experience against an +existing Foundry project. It covers the same lifecycle surfaces used by hosted +code agents and `invocations_ws` agents, plus the Voice wrapper and local Voice +dashboard. + +## Architecture under test + +```text +Voice client / local dashboard + | + | Voice realtime protocol + v +Voice wrapper (kind=voice, model_type=hosted_agent) + | + | Voice Bridge Protocol 1.0 over invocations_ws/1.0.0 + v +Hosted target (kind=hosted, user code) +``` + +Users write and deploy the hosted target code. The wrapper owns VAD, STT, TTS, +output voice, and Voice session settings. The wrapper and target must be in the +same Foundry project. + +## Prerequisites + +- A Foundry project in a region where Hosted Voice is enabled. West US 2 is the + recommended preview validation region. +- A model deployment that the hosted target can invoke. +- `az login` and `azd auth login` completed for the test subscription. +- A local build of the PR extension installed: + + ```powershell + cd cli/azd/extensions/azure.ai.agents + azd x build + ``` + +- A Hosted Voice target implementing Voice Bridge Protocol 1.0. The current + .NET sample is under `samples/voice-hosted-agent/voice-hosted-agent-dotnet` in + the `voice-first-agent-dev` repository. + +## Manifest + +Initialize an empty folder from the standard public Agent Manifest: + +```powershell +azd ai agent init -m +``` + +When source already exists, run `azd ai agent init` without `-m` from the source +directory. azd detects the local manifest and reuses a parent azd project when +present. + +Use one project service, one hosted target, and one Voice wrapper. The wrapper +references the target by its `azure.yaml` service name, not by a remote agent +name copied into the file. + +```yaml +services: + ai-project: + host: azure.ai.project + + voice-target: + host: azure.ai.agent + project: ./src/voice-target + language: csharp + kind: hosted + name: voice-target + uses: + - ai-project + metadata: + voiceLiveCompatible: "true" + bridgeProtocolVersion: "1.0" + protocols: + - protocol: invocations_ws + version: 1.0.0 + codeConfiguration: + runtime: dotnet_10 + entryPoint: VoiceHostedAgent.dll + dependencyResolution: bundled + container: + resources: + cpu: "1" + memory: 2Gi + + voice: + host: azure.ai.agent + kind: prompt-voice + name: voice + uses: + - ai-project + - voice-target + modelType: hosted_agent + targetAgent: + service: voice-target + version: deployed + store: false + audio: + output: + voice: + type: azure_standard + name: en-US-JennyNeural +``` + +`version: deployed` pins the wrapper to the target version recorded by the +current azd environment. Floating latest is intentionally not supported. + +## Automated local checks + +Run from `cli/azd/extensions/azure.ai.agents`: + +```powershell +go test ./... +go vet ./... +azd x build +``` + +Expected: all commands succeed. + +## Package and publish + +Run from the test azd project: + +```powershell +azd package --all +azd publish --all +``` + +Expected: + +- the hosted target builds and produces a code ZIP or container artifact; +- the project and Voice wrapper report no package artifact; +- publish succeeds without attempting to publish wrapper code. + +This matches the existing split between a code agent service and a declarative +service resource. + +## Deploy + +```powershell +azd deploy --all --no-prompt +``` + +Expected ordering: + +1. project dependency is ready; +2. target is packaged and deployed; +3. target version becomes active; +4. wrapper validates the target; +5. wrapper is created or updated through the unified Voice API. + +Expected target validation: + +- same Foundry project; +- `kind=hosted`; +- status `active`; +- `invocations_ws/1.0.0`; +- metadata `voiceLiveCompatible=true`; +- metadata `bridgeProtocolVersion=1.0`. + +Expected output includes the target `invocations_ws` endpoint and wrapper Voice +endpoint. + +## Environment outputs + +```powershell +azd env get-values +``` + +Expected target values: + +```text +AGENT__NAME +AGENT__VERSION +AGENT__PROJECT_ENDPOINT +AGENT__INVOCATIONS_WS_ENDPOINT +``` + +Expected wrapper values: + +```text +AGENT__NAME +AGENT__VERSION +AGENT__PROJECT_ENDPOINT +AGENT__ENDPOINT +AGENT__TARGET_NAME +AGENT__TARGET_VERSION +``` + +The wrapper endpoint is the end-user Voice realtime endpoint. The target +`invocations_ws` endpoint is a diagnostic/developer endpoint. + +## Show and doctor + +```powershell +azd ai agent show voice-target --output json +azd ai agent show voice --output json +azd ai agent doctor --output json +``` + +Expected: + +- target show returns an active hosted definition and `invocations_ws` endpoint; +- wrapper show returns `kind=voice`, `model_type=hosted_agent`, and the pinned + target name/version; +- doctor reports no failed checks. + +`doctor` is project-wide and does not currently accept a service argument. + +## Repeat and independent deployment + +Run the wrapper deployment twice: + +```powershell +azd deploy voice --no-prompt +azd deploy voice --no-prompt +``` + +Expected: the first command creates or updates the wrapper and the second uses +the unified update path. Both preserve a working endpoint. + +The wrapper can be managed independently after its target is deployed: + +```powershell +azd ai agent delete voice --force --no-prompt +azd deploy voice --no-prompt +``` + +Expected: + +- delete removes only the wrapper and clears its environment markers; +- the target remains active; +- the single-service deploy recreates only the wrapper. + +Do not delete the target before its managed wrapper. Full reverse-order +cleanup and ownership-aware `azd down` behavior are planned follow-up work. + +## Direct target protocol smoke + +Use a Voice Bridge Protocol client against: + +```text +AGENT__INVOCATIONS_WS_ENDPOINT +``` + +Expected frame sequence includes `session.ready`, response text events, and +`response.done`. This isolates target code and model access from the Voice +wrapper. It is diagnostic only and is not the end-user path. + +Generic `azd ai agent invoke` intentionally rejects an `invocations_ws`-only +target because the command supports `responses`, `invocations`, and `a2a`. + +## Local Voice dashboard + +From the `voice-first-agent-dev` repository: + +```powershell +cd tests/voice-agent-tests/voice-agents-tests-dashboard/voice_demo +python -m pip install -r requirements.txt +python demo_server.py ` + --backend "/" ` + --bind 127.0.0.1 ` + --port 9527 ` + --auto-delete false +``` + +Open `http://127.0.0.1:9527` on the same machine. + +Confirm: + +1. Agent backend is the intended Foundry project. +2. Inference modes include **Hosted agent**. +3. Agent discovery includes both the hosted target and Voice wrapper. +4. The connection dropdown selects the wrapper, not the target. This is by + design: Voice clients connect to the wrapper. +5. The wrapper details show the expected target name/version. +6. Connect and send a typed turn; text and audio should return. +7. With a microphone available, send a spoken turn and verify STT, target text, + and output audio. + +If the UI reports `getUserMedia ... Requested device not found`, the remote +desktop/browser has no microphone. This does not indicate an agent failure. + +## Negative tests + +Run these against a disposable environment and restore every value after the +test. + +| Case | Expected failure | +|---|---| +| Explicit `AZURE_VOICE_AGENT_API=legacy` | Hosted wrapper requires unified-flat | +| Target version marker points to a missing version | Remote target validation fails before wrapper mutation | +| Target project marker differs from the wrapper project | Dependency validation rejects cross-project binding | +| Target status is not active | Compatibility validation rejects it | +| Target lacks `invocations_ws/1.0.0` | Compatibility validation rejects it | +| Target lacks Voice Bridge metadata | Compatibility validation rejects it | +| Wrapper includes model/instructions/tools/handoff | Manifest validation rejects target-owned fields | +| Wrapper does not list target under `uses` | Target service resolution fails | + +After negative testing, run a normal wrapper deploy and one text turn to verify +the environment was restored. + +## Experience alignment matrix + +| Capability | Hosted/code agent | `invocations_ws` agent | Hosted Voice in this PR | +|---|---|---|---| +| `azure.ai.agent` service | Yes | Yes | Yes, target and wrapper | +| Project reuse/provision | Yes | Yes | Unchanged | +| Package/publish target code | Yes | Yes | Unchanged | +| Service graph deployment | Yes | Yes | Yes, `uses` orders target then wrapper | +| Single-service deploy | Yes | Yes | Yes | +| Remote version pinning | Yes | Yes | Wrapper pins deployed target version | +| `show` | Yes | Yes | Yes for both layers | +| `doctor` | Yes | Yes | Project-wide checks pass | +| Generic `invoke` | Responses/invocations | Not for WS | Not for Voice/WS; use Voice client/UI | +| Delete agent | Yes | Yes | Yes for each layer; wrapper first | +| `azd down` ownership cleanup | Existing behavior | Existing behavior | Follow-up for wrapper reverse cleanup | +| `azd ai agent init` scaffold | Yes | Yes | Yes, interactive and no-prompt CI paths | + +## Evidence to record + +For manual sign-off, record: + +- date/time and tester; +- PR commit and installed extension version; +- subscription, region, account, project, and endpoint; +- target and wrapper names/versions; +- output of test/build/package/publish/deploy/show/doctor; +- dashboard screenshot showing Hosted mode and wrapper target binding; +- typed response text; +- spoken input transcript and number/presence of returned audio frames; +- negative test results; +- cleanup status. diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/init.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/init.go index a5ea92150ad..f2a4dd02841 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/init.go @@ -371,7 +371,6 @@ func resolveAgentNameFromManifestPointer( flags.agentName = validated return validated, nil } - peeked := peekManifestName(ctx, manifestPointer, httpClient) if peeked == "" { // Defer to the inner flow which has access to the fully-loaded manifest. @@ -787,6 +786,7 @@ func synthesizeImageManifestFile(agentName, image string, flagProtocols []string // kindFlagPromptVoice is the accepted --kind value for a declarative voice agent. const kindFlagPromptVoice = "prompt-voice" +const kindFlagHostedVoice = "hosted-voice" // synthesizeVoiceManifestFile writes a temporary declarative (managed) voice // agent manifest (kind: prompt-voice) to a temp dir and returns its path plus a @@ -1163,6 +1163,8 @@ func agentDefiningFlagsSet(flags *initFlags, srcBlocksReuse bool) bool { flags.modelDeployment != "" || flags.projectResourceId != "" || flags.image != "" || + flags.kind != "" || + flags.voice != "" || srcBlocksReuse || len(flags.protocols) > 0 } @@ -1333,11 +1335,12 @@ from code-deploy ZIP packaging (uses .gitignore syntax).`, // when the other runs first (e.g. --kind prompt-voice --image would // otherwise silently create a hosted image agent). if flags.kind != "" { - if !strings.EqualFold(flags.kind, kindFlagPromptVoice) { + if !strings.EqualFold(flags.kind, kindFlagPromptVoice) && + !strings.EqualFold(flags.kind, kindFlagHostedVoice) { return exterrors.Validation( exterrors.CodeInvalidParameter, fmt.Sprintf("unsupported --kind value %q", flags.kind), - fmt.Sprintf("the only supported --kind value is %q", kindFlagPromptVoice), + fmt.Sprintf("supported --kind values are %q and %q", kindFlagPromptVoice, kindFlagHostedVoice), ) } if !promptVoicePreviewEnabled() { @@ -1347,14 +1350,28 @@ from code-deploy ZIP packaging (uses .gitignore syntax).`, fmt.Sprintf("set %s=true to enable prompt voice init", promptVoicePreviewEnvVar), ) } - if flags.image != "" { + if strings.EqualFold(flags.kind, kindFlagHostedVoice) && flags.image != "" { + return exterrors.Validation( + exterrors.CodeInvalidParameter, + "--kind hosted-voice cannot be combined with --image", + "hosted voice init requires local Voice Bridge source code; drop --image", + ) + } + if strings.EqualFold(flags.kind, kindFlagHostedVoice) && flags.manifestPointer != "" { + return exterrors.Validation( + exterrors.CodeInvalidParameter, + "--kind hosted-voice cannot be combined with --manifest", + "hosted voice init generates the target and wrapper from local code; drop --manifest", + ) + } + if strings.EqualFold(flags.kind, kindFlagPromptVoice) && flags.image != "" { return exterrors.Validation( exterrors.CodeInvalidParameter, "--kind prompt-voice cannot be combined with --image", "a voice agent is managed and has no container image; drop --image", ) } - if flags.manifestPointer != "" { + if strings.EqualFold(flags.kind, kindFlagPromptVoice) && flags.manifestPointer != "" { return exterrors.Validation( exterrors.CodeInvalidParameter, "--kind prompt-voice cannot be combined with --manifest", @@ -1400,7 +1417,7 @@ from code-deploy ZIP packaging (uses .gitignore syntax).`, // language prompts and code scaffolding). Mirrors the --image fast path. // --kind value and --image incompatibility are validated above, before // either synthesis branch. - if flags.kind != "" && flags.manifestPointer == "" { + if strings.EqualFold(flags.kind, kindFlagPromptVoice) && flags.manifestPointer == "" { if flags.agentName == "" { return exterrors.Validation( exterrors.CodeInvalidParameter, @@ -1690,7 +1707,13 @@ from code-deploy ZIP packaging (uses .gitignore syntax).`, } } else { // No manifest provided - prompt user for init mode - initMode, err := promptInitMode(ctx, azdClient, flags.noPrompt) + initMode := "" + var err error + if strings.EqualFold(flags.kind, kindFlagHostedVoice) { + initMode = initModeHostedVoice + } else { + initMode, err = promptInitMode(ctx, azdClient, flags.noPrompt) + } if err != nil { if exterrors.IsCancellation(err) { return exterrors.Cancelled("initialization was cancelled") @@ -1857,6 +1880,20 @@ from code-deploy ZIP packaging (uses .gitignore syntax).`, return err } + case initModeHostedVoice: + flags.kind = kindFlagHostedVoice + action := &InitFromCodeAction{ + azdClient: azdClient, + flags: flags, + httpClient: httpClient, + } + if err := action.Run(ctx); err != nil { + if exterrors.IsCancellation(err) { + return exterrors.Cancelled("initialization was cancelled") + } + return err + } + default: // initModeFromCode - use existing code in current directory action := &InitFromCodeAction{ @@ -1927,9 +1964,8 @@ from code-deploy ZIP packaging (uses .gitignore syntax).`, "Incompatible with --deploy-mode code.") cmd.Flags().StringVar(&flags.kind, "kind", "", - "Agent kind to initialize non-interactively. Currently supports 'prompt-voice' to create a "+ - "declarative (managed) voice agent, skipping template/language selection and code scaffolding. "+ - "Use --model to name the speech-to-speech model and --voice to set the output voice.") + "Agent kind to initialize. Supports 'prompt-voice' and 'hosted-voice'. "+ + "Hosted voice uses local code for a Voice Bridge target and creates a Voice wrapper.") cmd.Flags().StringVar(&flags.voice, "voice", "", "Output voice name for private prompt-voice automation. Hidden until public preview.") @@ -2006,6 +2042,14 @@ func (a *InitAction) Run(ctx context.Context) error { if err != nil { return fmt.Errorf("downloading agent.yaml: %w", err) } + if _, hostedVoice, err := hostedVoiceManifestTarget(agentManifest); err != nil { + return err + } else if hostedVoice { + a.flags.kind = kindFlagHostedVoice + if err := validateHostedVoiceServiceNamesForProject(ctx, a.azdClient, a.serviceNameOverride); err != nil { + return err + } + } // Prompt for deploy mode (code vs container) for hosted agents. // Code deploy is supported for Python and .NET projects. @@ -2019,10 +2063,27 @@ func (a *InitAction) Run(ctx context.Context) error { if a.isCodeDeploy { // Prompt for code configuration and update the manifest - codeConfig, err := promptCodeConfig(ctx, a.azdClient, targetDir, a.flags.noPrompt, codeDeployOptions{ + codeOptions := codeDeployOptions{ runtime: a.flags.runtime, entryPoint: a.flags.entryPoint, depResolution: a.flags.depResolution, + } + if hostedAgent, ok := agentManifest.Template.(agent_yaml.ContainerAgent); ok && + hostedAgent.CodeConfiguration != nil { + if codeOptions.runtime == "" { + codeOptions.runtime = hostedAgent.CodeConfiguration.Runtime + } + if codeOptions.entryPoint == "" { + codeOptions.entryPoint = hostedAgent.CodeConfiguration.EntryPoint + } + if codeOptions.depResolution == "" && hostedAgent.CodeConfiguration.DependencyResolution != nil { + codeOptions.depResolution = *hostedAgent.CodeConfiguration.DependencyResolution + } + } + codeConfig, err := promptCodeConfig(ctx, a.azdClient, targetDir, a.flags.noPrompt, codeDeployOptions{ + runtime: codeOptions.runtime, + entryPoint: codeOptions.entryPoint, + depResolution: codeOptions.depResolution, }, a.userProvidedManifest) if err != nil { return fmt.Errorf("prompting for code configuration: %w", err) @@ -2117,6 +2178,11 @@ func (a *InitAction) Run(ctx context.Context) error { if err := a.addToProject(ctx, targetDir, agentManifest); err != nil { return fmt.Errorf("failed to add agent to azure.yaml: %w", err) } + if strings.EqualFold(a.flags.kind, kindFlagHostedVoice) { + if err := addHostedVoiceWrapperToProject(ctx, a.azdClient, a.serviceNameOverride); err != nil { + return fmt.Errorf("adding hosted voice wrapper: %w", err) + } + } // Run post-init validations (advisory warnings only) if ca, ok := agentManifest.Template.(agent_yaml.ContainerAgent); ok { @@ -4449,8 +4515,15 @@ func (a *InitAction) validateCodeDeployFlags() error { if err := validateImageFlag(a.flags.image, a.flags.deployMode); err != nil { return err } + noPrompt := a.flags.noPrompt + if a.flags.manifestPointer != "" { + // A standard manifest can provide runtime, entry point, and dependency + // resolution. Validate values now and enforce completeness after the + // manifest has been loaded and merged with explicit CLI overrides. + noPrompt = false + } return validateCodeDeployInput( - a.flags.noPrompt, a.flags.deployMode, a.flags.runtime, a.flags.entryPoint, a.flags.depResolution) + noPrompt, a.flags.deployMode, a.flags.runtime, a.flags.entryPoint, a.flags.depResolution) } var initImageRefRe = regexp.MustCompile( diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_code.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_code.go index e2ed85ac68f..88e41a8fe2a 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_code.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_code.go @@ -84,7 +84,8 @@ func (a *InitFromCodeAction) Run(ctx context.Context) error { // Guard against silently overwriting an existing agent definition. Reached // when the user declined the reuse prompt in RunE or bypassed it; we still // refuse in --no-prompt and confirm interactively. - if existing, statErr := findExistingAgentYaml(srcDir); statErr == nil && existing != "" { + if existing, statErr := findExistingAgentYaml(srcDir); statErr == nil && existing != "" && + !strings.EqualFold(a.flags.kind, kindFlagHostedVoice) { displayPath, relErr := filepath.Rel(srcDir, existing) if relErr != nil || displayPath == "" { displayPath = existing @@ -303,10 +304,25 @@ func (a *InitFromCodeAction) createDefinitionFromLocalAgent(ctx context.Context) } } - // Prompt user for supported protocols - protocols, err := promptProtocols(ctx, a.azdClient.Prompt(), a.flags.noPrompt, a.flags.protocols) - if err != nil { - return nil, err + // Hosted Voice targets implement the Voice Bridge 1.0 contract over the + // invocations_ws/1.0.0 transport. Other hosted agents retain the normal + // protocol selection and current default versions. + var protocols []agent_yaml.ProtocolVersionRecord + if strings.EqualFold(a.flags.kind, kindFlagHostedVoice) { + if len(a.flags.protocols) > 0 && + !(len(a.flags.protocols) == 1 && strings.EqualFold(a.flags.protocols[0], "invocations_ws")) { + return nil, exterrors.Validation( + exterrors.CodeInvalidParameter, + "hosted voice targets require the invocations_ws protocol", + "omit --protocol or pass --protocol invocations_ws", + ) + } + protocols = []agent_yaml.ProtocolVersionRecord{{Protocol: "invocations_ws", Version: "1.0.0"}} + } else { + protocols, err = promptProtocols(ctx, a.azdClient.Prompt(), a.flags.noPrompt, a.flags.protocols) + if err != nil { + return nil, err + } } // Step 1: Foundry project selection @@ -547,6 +563,20 @@ func (a *InitFromCodeAction) createDefinitionFromLocalAgent(ctx context.Context) Protocols: protocols, CodeConfiguration: codeConfig, } + if strings.EqualFold(a.flags.kind, kindFlagHostedVoice) { + definition.Metadata = &map[string]any{ + "voiceLiveCompatible": "true", + "bridgeProtocolVersion": "1.0", + } + definition.EnvironmentVariables = appendEnvVar(definition.EnvironmentVariables, agent_yaml.EnvironmentVariable{ + Name: "AZURE_OPENAI_ENDPOINT", + Value: "${FOUNDRY_PROJECT_ENDPOINT}/openai/v1/responses", + }) + definition.EnvironmentVariables = appendEnvVar(definition.EnvironmentVariables, agent_yaml.EnvironmentVariable{ + Name: "AZURE_OPENAI_DEPLOYMENT", + Value: "${AZURE_AI_MODEL_DEPLOYMENT_NAME}", + }) + } // An activity agent additionally advertises the friendly "activity" endpoint // guarded by BotServiceRbac. We compose this into any existing agent_endpoint @@ -792,6 +822,12 @@ func (a *InitFromCodeAction) addToProject( isCodeDeploy bool, ) error { agentName := definition.Name + agentServiceName := strings.ReplaceAll(agentName, " ", "") + if strings.EqualFold(a.flags.kind, kindFlagHostedVoice) { + if err := a.validateHostedVoiceServiceNames(ctx, agentServiceName); err != nil { + return err + } + } // If targetDir is ".", resolve the actual relative path from the project root to cwd. // This ensures azure.yaml gets the correct "project:" value when init is run from a subdirectory. if targetDir == "." { @@ -811,6 +847,10 @@ func (a *InitFromCodeAction) addToProject( Cpu: project.DefaultCpu, }, } + if strings.EqualFold(a.flags.kind, kindFlagHostedVoice) { + agentConfig.Container.Resources.Cpu = "1" + agentConfig.Container.Resources.Memory = "2Gi" + } agentConfig.Deployments = a.deploymentDetails @@ -845,7 +885,6 @@ func (a *InitFromCodeAction) addToProject( language = "csharp" } - agentServiceName := strings.ReplaceAll(agentName, " ", "") serviceConfig := &azdext.ServiceConfig{ Name: agentServiceName, RelativePath: targetDir, @@ -895,10 +934,115 @@ func (a *InitFromCodeAction) addToProject( return err } + if strings.EqualFold(a.flags.kind, kindFlagHostedVoice) { + if err := a.addHostedVoiceWrapper(ctx, agentServiceName); err != nil { + return err + } + } + printAgentAddedMessage(agentName) return nil } +func (a *InitFromCodeAction) validateHostedVoiceServiceNames(ctx context.Context, targetServiceName string) error { + return validateHostedVoiceServiceNamesForProject(ctx, a.azdClient, targetServiceName) +} + +func validateHostedVoiceServiceNamesForProject( + ctx context.Context, + azdClient *azdext.AzdClient, + targetServiceName string, +) error { + response, err := azdClient.Project().Get(ctx, &azdext.EmptyRequest{}) + if err != nil { + return fmt.Errorf("checking existing services for hosted voice init: %w", err) + } + if response.Project == nil { + return nil + } + for _, serviceName := range []string{targetServiceName, hostedVoiceWrapperName(targetServiceName)} { + if _, exists := response.Project.Services[serviceName]; exists { + return exterrors.Validation( + exterrors.CodeInvalidParameter, + fmt.Sprintf("service %q already exists", serviceName), + "choose a different target agent name so target and wrapper service names are unique", + ) + } + } + return nil +} + +func (a *InitFromCodeAction) addHostedVoiceWrapper(ctx context.Context, targetServiceName string) error { + return addHostedVoiceWrapperToProject(ctx, a.azdClient, targetServiceName) +} + +func addHostedVoiceWrapperToProject( + ctx context.Context, + azdClient *azdext.AzdClient, + targetServiceName string, +) error { + wrapperName := hostedVoiceWrapperName(targetServiceName) + response, err := azdClient.Project().Get(ctx, &azdext.EmptyRequest{}) + if err != nil || response.Project == nil { + return fmt.Errorf("loading project services before adding hosted voice wrapper: %w", err) + } + if _, exists := response.Project.Services[wrapperName]; exists { + return exterrors.Validation( + exterrors.CodeInvalidParameter, + fmt.Sprintf("hosted voice wrapper service %q already exists", wrapperName), + "choose a different target agent name so the generated wrapper service name is unique", + ) + } + projectServiceName := existingProjectServiceKey(ctx, azdClient) + if projectServiceName == "" { + return fmt.Errorf("cannot resolve the azure.ai.project service for hosted voice wrapper %q", wrapperName) + } + store := false + description := "Voice wrapper for hosted target " + targetServiceName + voiceAgent := agent_yaml.VoiceAgent{ + AgentDefinition: agent_yaml.AgentDefinition{ + Kind: agent_yaml.AgentKindPromptVoice, + Name: wrapperName, + Description: &description, + }, + ModelType: agent_yaml.VoiceModelTypeHostedAgent, + TargetAgent: &agent_yaml.VoiceTargetAgent{ + Service: targetServiceName, + Version: "deployed", + }, + Store: &store, + } + props, err := project.VoiceAgentDefinitionToServiceProperties(voiceAgent, nil) + if err != nil { + return err + } + if _, err := azdClient.Project().AddService(ctx, &azdext.AddServiceRequest{Service: &azdext.ServiceConfig{ + Name: wrapperName, + Host: AiAgentHost, + AdditionalProperties: props, + }}); err != nil { + return fmt.Errorf("adding hosted voice wrapper service: %w", err) + } + + if err := setServiceUses(ctx, azdClient, wrapperName, []string{projectServiceName, targetServiceName}); err != nil { + return err + } + fmt.Printf(" %s Added hosted voice wrapper %s -> %s\n", color.GreenString("+"), wrapperName, targetServiceName) + return nil +} + +func hostedVoiceWrapperName(targetServiceName string) string { + const suffix = "-voice" + base := strings.TrimRight(targetServiceName, "-") + if len(base)+len(suffix) > 63 { + base = strings.TrimRight(base[:63-len(suffix)], "-") + } + if base == "" { + base = "agent" + } + return base + suffix +} + // promptCodeConfiguration prompts the user for code deploy configuration settings. func (a *InitFromCodeAction) promptCodeConfiguration(ctx context.Context, srcDir string) (*agent_yaml.CodeConfiguration, error) { return promptCodeConfig(ctx, a.azdClient, srcDir, a.flags.noPrompt, codeDeployOptions{ diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_code_test.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_code_test.go index dd28c798dab..a62531b3220 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_code_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_code_test.go @@ -5,6 +5,7 @@ package cmd import ( "azureaiagent/internal/pkg/agents/agent_yaml" + "azureaiagent/internal/project" "context" "os" "path/filepath" @@ -126,6 +127,38 @@ func TestSanitizeAgentName(t *testing.T) { } } +func TestHostedVoiceWrapperName(t *testing.T) { + t.Parallel() + require.Equal(t, "voice-target-voice", hostedVoiceWrapperName("voice-target")) + long := strings.Repeat("a", 63) + got := hostedVoiceWrapperName(long) + require.Len(t, got, 63) + require.True(t, strings.HasSuffix(got, "-voice")) +} + +func TestAddHostedVoiceWrapper(t *testing.T) { + server := &recordingProjectServer{existing: map[string]*azdext.ServiceConfig{ + "foundry-project": {Name: "foundry-project", Host: AiProjectHost}, + }} + client := newProjectRecorderClient(t, server) + action := &InitFromCodeAction{azdClient: client} + + require.NoError(t, action.addHostedVoiceWrapper(t.Context(), "voice-target")) + + server.mu.Lock() + defer server.mu.Unlock() + require.Len(t, server.added, 1) + wrapper := server.added[0] + require.Equal(t, "voice-target-voice", wrapper.Name) + voiceAgent, found, err := project.VoiceAgentFromResolvedService(wrapper, t.TempDir()) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, agent_yaml.VoiceModelTypeHostedAgent, voiceAgent.ModelType) + require.Equal(t, "voice-target", voiceAgent.TargetAgent.Service) + require.Equal(t, "deployed", voiceAgent.TargetAgent.Version) + require.ElementsMatch(t, []string{"foundry-project", "voice-target"}, server.uses[wrapper.Name]) +} + func TestNormalizeForFuzzyMatch(t *testing.T) { t.Parallel() diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_templates_helpers.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_templates_helpers.go index 305ce1ba9b5..d5e8429ab8b 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_templates_helpers.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_templates_helpers.go @@ -107,6 +107,9 @@ const ( // (managed) voice agent. It maps to the same synthesized-manifest fast path // as `azd ai agent init --kind prompt-voice`. initModeVoice = "prompt_voice" + // initModeHostedVoice creates a Voice Bridge hosted target from local code + // plus a declarative Voice wrapper that references the target service. + initModeHostedVoice = "hosted_voice" ) // voiceInitChoice is the interactive menu entry for creating a prompt voice agent. @@ -117,6 +120,11 @@ var voiceInitChoice = &azdext.SelectChoice{ Value: initModeVoice, } +var hostedVoiceInitChoice = &azdext.SelectChoice{ + Label: "Create a hosted voice agent from the code in the current directory", + Value: initModeHostedVoice, +} + // promptInitMode asks the user whether to use existing code, start from a // template, or create a prompt voice agent. // If the current directory is empty, the "use existing code" option is omitted @@ -157,6 +165,9 @@ func promptInitMode(ctx context.Context, azdClient *azdext.AzdClient, noPrompt b } if voicePreviewEnabled { choices = append(choices, voiceInitChoice) + if !empty { + choices = append(choices, hostedVoiceInitChoice) + } } defaultIndex := int32(0) diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_templates_helpers_test.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_templates_helpers_test.go index 117ec3e8f35..964f581e3db 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_templates_helpers_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_from_templates_helpers_test.go @@ -282,6 +282,26 @@ func TestPromptInitMode_ShowsVoiceChoiceWhenPreviewEnabled(t *testing.T) { require.Equal(t, "Create a prompt voice agent", prompts.lastSelect.Options.Choices[1].Label) } +func TestPromptInitMode_ShowsHostedVoiceChoiceForLocalCode(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + t.Setenv(promptVoicePreviewEnvVar, "true") + require.NoError(t, os.WriteFile(filepath.Join(dir, "Program.cs"), []byte("class Program {}\n"), 0600)) + + prompts := &helpersPromptServer{selectIndex: 3} + azdClient := newHelpersTestAzdClient(t, &helpersProjectServer{}, prompts) + + mode, err := promptInitMode(t.Context(), azdClient, false) + + require.NoError(t, err) + require.Equal(t, initModeHostedVoice, mode) + require.NotNil(t, prompts.lastSelect) + require.Len(t, prompts.lastSelect.Options.Choices, 4) + require.Equal(t, + "Create a hosted voice agent from the code in the current directory", + prompts.lastSelect.Options.Choices[3].Label) +} + func TestFindRecommendedIndex(t *testing.T) { t.Parallel() diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_hosted_voice.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_hosted_voice.go new file mode 100644 index 00000000000..1f49781dfe3 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_hosted_voice.go @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + "strings" + + "azureaiagent/internal/pkg/agents/agent_yaml" + + "gopkg.in/yaml.v3" +) + +func hostedVoiceManifestTarget(manifest *agent_yaml.AgentManifest) (*agent_yaml.ContainerAgent, bool, error) { + if manifest == nil { + return nil, false, nil + } + templateYAML, err := yaml.Marshal(manifest.Template) + if err != nil { + return nil, false, fmt.Errorf("marshaling hosted voice manifest template: %w", err) + } + var target agent_yaml.ContainerAgent + if err := yaml.Unmarshal(templateYAML, &target); err != nil { + return nil, false, nil + } + if target.Kind != agent_yaml.AgentKindHosted { + return nil, false, nil + } + compatibleProtocol := false + for _, protocol := range target.Protocols { + if protocol.Protocol == "invocations_ws" && protocol.Version == "1.0.0" { + compatibleProtocol = true + break + } + } + if !compatibleProtocol || target.Metadata == nil { + return nil, false, nil + } + metadata := *target.Metadata + voiceCompatible := strings.EqualFold(strings.TrimSpace(fmt.Sprint(metadata["voiceLiveCompatible"])), "true") + bridgeVersion := strings.TrimSpace(fmt.Sprint(metadata["bridgeProtocolVersion"])) + if !voiceCompatible || bridgeVersion != "1.0" { + return nil, false, nil + } + return &target, true, nil +} diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_hosted_voice_test.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_hosted_voice_test.go new file mode 100644 index 00000000000..0d9135e969f --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_hosted_voice_test.go @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaiagent/internal/pkg/agents/agent_yaml" + + "github.com/stretchr/testify/require" +) + +func TestHostedVoiceManifestTarget(t *testing.T) { + t.Parallel() + metadata := map[string]any{ + "voiceLiveCompatible": "true", + "bridgeProtocolVersion": "1.0", + } + manifest := &agent_yaml.AgentManifest{Template: agent_yaml.ContainerAgent{ + AgentDefinition: agent_yaml.AgentDefinition{ + Kind: agent_yaml.AgentKindHosted, Name: "voice-target", Metadata: &metadata, + }, + Protocols: []agent_yaml.ProtocolVersionRecord{{Protocol: "invocations_ws", Version: "1.0.0"}}, + }} + target, compatible, err := hostedVoiceManifestTarget(manifest) + require.NoError(t, err) + require.True(t, compatible) + require.Equal(t, "voice-target", target.Name) +} + +func TestHostedVoiceManifestTargetRejectsIncompatibleManifest(t *testing.T) { + t.Parallel() + metadata := map[string]any{ + "voiceLiveCompatible": "true", + "bridgeProtocolVersion": "2.0", + } + manifest := &agent_yaml.AgentManifest{Template: agent_yaml.ContainerAgent{ + AgentDefinition: agent_yaml.AgentDefinition{ + Kind: agent_yaml.AgentKindHosted, Name: "voice-target", Metadata: &metadata, + }, + Protocols: []agent_yaml.ProtocolVersionRecord{{Protocol: "invocations_ws", Version: "1.0.0"}}, + }} + _, compatible, err := hostedVoiceManifestTarget(manifest) + require.NoError(t, err) + require.False(t, compatible) +} + +func TestHostedVoiceManifestTargetRejectsGenericInvocationsWS(t *testing.T) { + t.Parallel() + manifest := &agent_yaml.AgentManifest{Template: agent_yaml.ContainerAgent{ + AgentDefinition: agent_yaml.AgentDefinition{Kind: agent_yaml.AgentKindHosted, Name: "generic"}, + Protocols: []agent_yaml.ProtocolVersionRecord{{Protocol: "invocations_ws", Version: "1.0.0"}}, + }} + _, compatible, err := hostedVoiceManifestTarget(manifest) + require.NoError(t, err) + require.False(t, compatible) +} diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_test.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_test.go index 083ac34a231..441a8114820 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/init_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/init_test.go @@ -3039,6 +3039,13 @@ func TestCodeDeployFlagValidation(t *testing.T) { flags: initFlags{noPrompt: false, deployMode: "code"}, wantErr: false, }, + { + name: "no-prompt manifest can provide code configuration", + flags: initFlags{ + noPrompt: true, deployMode: "code", manifestPointer: "agent.manifest.yaml", + }, + wantErr: false, + }, { name: "invalid deploy-mode value fails", flags: initFlags{noPrompt: true, deployMode: "invalid"}, diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/listen.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/listen.go index f864a5d2b41..182105b5671 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/listen.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/listen.go @@ -24,6 +24,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/azure/azure-dev/cli/azd/pkg/azdext" "github.com/azure/azure-dev/cli/azd/pkg/output" + "google.golang.org/protobuf/proto" ) // configureExtensionHost wires the service target and event handlers on the @@ -503,11 +504,11 @@ func resolveAgentServiceConfigWithProjectOverrides( svc *azdext.ServiceConfig, projectRoot string, ) (*azdext.ServiceConfig, error) { - resolvedSvc := *svc - if err := project.ResolveServiceConfigInPlace(&resolvedSvc, projectRoot); err != nil { + resolvedSvc := proto.Clone(svc).(*azdext.ServiceConfig) + if err := project.ResolveServiceConfigInPlace(resolvedSvc, projectRoot); err != nil { return nil, err } - return &resolvedSvc, nil + return resolvedSvc, nil } func warnLegacySimpleTeamsArtifacts(proj *azdext.ProjectConfig, svc *azdext.ServiceConfig) { diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go index 86b452662d3..3cda58ced08 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go @@ -354,12 +354,19 @@ type VoiceModelType string const ( VoiceModelTypeManaged VoiceModelType = "managed" VoiceModelTypeSelfDeployed VoiceModelType = "self_deployed" + VoiceModelTypeHostedAgent VoiceModelType = "hosted_agent" ) +// VoiceTargetAgentReference pins a voice wrapper to a hosted agent version. +type VoiceTargetAgentReference struct { + Name string `json:"name"` + Version string `json:"version,omitempty"` +} + // VoiceAudioFormat describes a PCM audio stream format (e.g. audio/pcm @ 24 kHz). type VoiceAudioFormat struct { Type string `json:"type"` - Rate int `json:"rate"` + Rate *int `json:"rate,omitempty"` } // VoiceTurnDetection configures server-side voice-activity detection so the @@ -369,26 +376,47 @@ type VoiceTurnDetection struct { Threshold *float64 `json:"threshold,omitempty"` PrefixPaddingMs *int `json:"prefix_padding_ms,omitempty"` SilenceDurationMs *int `json:"silence_duration_ms,omitempty"` + CreateResponse *bool `json:"create_response,omitempty"` + Eagerness *string `json:"eagerness,omitempty"` + SpeechDurationMs *int `json:"speech_duration_ms,omitempty"` + RemoveFillerWords *bool `json:"remove_filler_words,omitempty"` + InterruptResponse *bool `json:"interrupt_response,omitempty"` + Languages []string `json:"languages,omitempty"` + AutoTruncate *bool `json:"auto_truncate,omitempty"` } // VoiceTranscription enables user-speech transcription events on the input stream. type VoiceTranscription struct { - Model string `json:"model,omitempty"` + Model string `json:"model,omitempty"` + Language *string `json:"language,omitempty"` + Prompt *string `json:"prompt,omitempty"` +} + +// VoiceNoiseReduction configures input audio noise reduction. +type VoiceNoiseReduction struct { + Type string `json:"type"` } // VoiceInputConfig is the input (caller -> agent) audio configuration. type VoiceInputConfig struct { - Format *VoiceAudioFormat `json:"format,omitempty"` - TurnDetection *VoiceTurnDetection `json:"turn_detection,omitempty"` - Transcription *VoiceTranscription `json:"transcription,omitempty"` + Format *VoiceAudioFormat `json:"format,omitempty"` + NoiseReduction *VoiceNoiseReduction `json:"noise_reduction,omitempty"` + EchoCancellation map[string]any `json:"echo_cancellation,omitempty"` + TurnDetection *VoiceTurnDetection `json:"turn_detection,omitempty"` + Transcription *VoiceTranscription `json:"transcription,omitempty"` } // VoiceConfig selects the output voice. Type is "openai" for realtime voices // (single lowercase word, e.g. "alloy") or "azure_standard" for Azure Neural // voices (e.g. "en-US-Ava:DragonHDLatestNeural"). type VoiceConfig struct { - Type string `json:"type"` - Name string `json:"name"` + Type string `json:"type"` + Name string `json:"name"` + Style *string `json:"style,omitempty"` + Pitch *string `json:"pitch,omitempty"` + Rate *string `json:"rate,omitempty"` + Locale *string `json:"locale,omitempty"` + Volume *string `json:"volume,omitempty"` } // VoiceOutputConfig is the output (agent -> caller) audio configuration for the @@ -399,6 +427,11 @@ type VoiceOutputConfig struct { Voice string `json:"voice,omitempty"` VoiceType string `json:"voice_type,omitempty"` VoiceLocale string `json:"voice_locale,omitempty"` + Style *string `json:"style,omitempty"` + Pitch *string `json:"pitch,omitempty"` + Rate *string `json:"rate,omitempty"` + Volume *string `json:"volume,omitempty"` + Speed *float64 `json:"speed,omitempty"` } // VoiceAudioConfig bundles the input and output audio configuration. @@ -411,12 +444,22 @@ type VoiceAudioConfig struct { // prompt voice agent. Its Kind is always AgentKindVoice ("voice"). type VoiceAgentDefinition struct { AgentDefinition - ModelType VoiceModelType `json:"model_type"` - Model string `json:"model"` - Instructions string `json:"instructions,omitempty"` - Audio *VoiceAudioConfig `json:"audio,omitempty"` - OutputModalities []string `json:"output_modalities,omitempty"` - Store *bool `json:"store,omitempty"` + ModelType VoiceModelType `json:"model_type"` + Model string `json:"model,omitempty"` + TargetAgent *VoiceTargetAgentReference `json:"target_agent,omitempty"` + Instructions string `json:"instructions,omitempty"` + StructuredInputs map[string]any `json:"structured_inputs,omitempty"` + Audio *VoiceAudioConfig `json:"audio,omitempty"` + OutputModalities []string `json:"output_modalities,omitempty"` + Store *bool `json:"store,omitempty"` + Tools []map[string]any `json:"tools,omitempty"` + Avatar map[string]any `json:"avatar,omitempty"` + Greeting map[string]any `json:"greeting,omitempty"` + Handoff map[string]any `json:"handoff,omitempty"` + ToolChoice any `json:"tool_choice,omitempty"` + ParallelToolCalls *bool `json:"parallel_tool_calls,omitempty"` + MaxOutputTokens any `json:"max_output_tokens,omitempty"` + Include []string `json:"include,omitempty"` } // CreateAgentVersionRequest represents a request to create an agent version diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go index cff9c5affe9..e9241d92630 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go @@ -594,7 +594,13 @@ func voiceWireType(voice *agent_api.VoiceConfig) string { } func voiceWireLocale(voice *agent_api.VoiceConfig) string { - if voice == nil || voice.Name == "" || isOpenAIVoice(voice.Name) { + if voice == nil || voice.Name == "" { + return "" + } + if voice.Locale != nil && strings.TrimSpace(*voice.Locale) != "" { + return strings.TrimSpace(*voice.Locale) + } + if isOpenAIVoice(voice.Name) { return "" } match := azureNeuralVoicePattern.FindStringSubmatch(voice.Name) @@ -604,34 +610,175 @@ func voiceWireLocale(voice *agent_api.VoiceConfig) string { return match[1] } +func defaultVoiceAudioFormat() *agent_api.VoiceAudioFormat { + rate := defaultVoiceAudioRate + return &agent_api.VoiceAudioFormat{Type: defaultVoiceAudioType, Rate: &rate} +} + +func mapVoiceAudioFormat(format *VoiceAudioFormat, fallback *agent_api.VoiceAudioFormat) *agent_api.VoiceAudioFormat { + out := &agent_api.VoiceAudioFormat{} + if fallback != nil { + *out = *fallback + } + if format != nil { + if strings.TrimSpace(format.Type) != "" { + out.Type = strings.TrimSpace(format.Type) + } + if format.Rate != nil { + out.Rate = format.Rate + } else if out.Type == "audio/pcmu" || out.Type == "audio/pcma" { + out.Rate = nil + } + } + return out +} + +func mapVoiceTurnDetection(turnDetection *VoiceTurnDetection) *agent_api.VoiceTurnDetection { + out := &agent_api.VoiceTurnDetection{Type: defaultVoiceTurnDetectionType} + if turnDetection == nil { + return out + } + if strings.TrimSpace(turnDetection.Type) != "" { + out.Type = strings.TrimSpace(turnDetection.Type) + } + out.Threshold = turnDetection.Threshold + out.PrefixPaddingMs = turnDetection.PrefixPaddingMs + out.SilenceDurationMs = turnDetection.SilenceDurationMs + out.CreateResponse = turnDetection.CreateResponse + out.Eagerness = turnDetection.Eagerness + out.SpeechDurationMs = turnDetection.SpeechDurationMs + out.RemoveFillerWords = turnDetection.RemoveFillerWords + out.InterruptResponse = turnDetection.InterruptResponse + out.Languages = turnDetection.Languages + out.AutoTruncate = turnDetection.AutoTruncate + return out +} + +func mapVoiceTranscription(transcription *VoiceTranscription) *agent_api.VoiceTranscription { + out := &agent_api.VoiceTranscription{Model: defaultVoiceInputTranscriptionModel} + if transcription == nil { + return out + } + if strings.TrimSpace(transcription.Model) != "" { + out.Model = strings.TrimSpace(transcription.Model) + } + out.Language = transcription.Language + out.Prompt = transcription.Prompt + return out +} + +func mapVoiceConfig(voice *VoiceConfig, fallbackName string) *agent_api.VoiceConfig { + if voice == nil { + return buildVoiceConfig(fallbackName) + } + name := strings.TrimSpace(voice.Name) + if name == "" { + name = fallbackName + } + voiceType := strings.TrimSpace(voice.Type) + if voiceType == "" { + out := buildVoiceConfig(name) + out.Style = voice.Style + out.Pitch = voice.Pitch + out.Rate = voice.Rate + out.Locale = voice.Locale + out.Volume = voice.Volume + return out + } + if voiceType == "openai" { + name = strings.ToLower(name) + } + return &agent_api.VoiceConfig{ + Type: voiceType, + Name: name, + Style: voice.Style, + Pitch: voice.Pitch, + Rate: voice.Rate, + Locale: voice.Locale, + Volume: voice.Volume, + } +} + +func mapVoiceStructuredInputs(inputs map[string]any) map[string]any { + if len(inputs) == 0 { + return nil + } + out := make(map[string]any, len(inputs)) + for name, input := range inputs { + inputMap, ok := input.(map[string]any) + if !ok { + out[name] = input + continue + } + + mapped := maps.Clone(inputMap) + if value, ok := mapped["defaultValue"]; ok { + if _, hasSnakeCase := mapped["default_value"]; !hasSnakeCase { + mapped["default_value"] = value + } + delete(mapped, "defaultValue") + } + out[name] = mapped + } + return out +} + // CreateVoiceAgentAPIRequest builds a CreateAgentRequest for a declarative // voice agent. It translates the authoring kind "prompt-voice" into the // data-plane service kind "voice" and defaults the audio pipeline. func CreateVoiceAgentAPIRequest(voiceAgent VoiceAgent) (*agent_api.CreateAgentRequest, error) { - return createVoiceAgentAPIRequest(voiceAgent) + return createVoiceAgentAPIRequest(voiceAgent, nil) } -func createVoiceAgentAPIRequest(voiceAgent VoiceAgent) (*agent_api.CreateAgentRequest, error) { - modelID := "" - if voiceAgent.Model != nil { - modelID = strings.TrimSpace(voiceAgent.Model.Id) - } - if modelID == "" { - return nil, fmt.Errorf("model.id is required for a prompt-voice agent") - } +// CreateHostedVoiceAgentAPIRequest builds a strict hosted-agent voice wrapper +// around the deployed target resolved by the project layer. +func CreateHostedVoiceAgentAPIRequest( + voiceAgent VoiceAgent, + target agent_api.VoiceTargetAgentReference, +) (*agent_api.CreateAgentRequest, error) { + return createVoiceAgentAPIRequest(voiceAgent, &target) +} +func createVoiceAgentAPIRequest( + voiceAgent VoiceAgent, + target *agent_api.VoiceTargetAgentReference, +) (*agent_api.CreateAgentRequest, error) { modelType := agent_api.VoiceModelTypeManaged if voiceAgent.ModelType != "" { modelType = agent_api.VoiceModelType(voiceAgent.ModelType) } - if modelType != agent_api.VoiceModelTypeManaged && modelType != agent_api.VoiceModelTypeSelfDeployed { + hostedAgent := modelType == agent_api.VoiceModelTypeHostedAgent + if hostedAgent { + if target == nil || strings.TrimSpace(target.Name) == "" || strings.TrimSpace(target.Version) == "" { + return nil, fmt.Errorf("resolved target agent name and version are required when model_type is 'hosted_agent'") + } + if voiceAgent.Model != nil || voiceAgent.Instructions != nil || len(voiceAgent.StructuredInputs) > 0 || + len(voiceAgent.Tools) > 0 || voiceAgent.ToolChoice != nil || voiceAgent.ParallelToolCalls != nil || + voiceAgent.MaxOutputTokens != nil || len(voiceAgent.Include) > 0 || len(voiceAgent.Handoff) > 0 { + return nil, fmt.Errorf("model, instructions, structured_inputs, tools, tool_choice, parallel_tool_calls, max_output_tokens, include, and handoff belong to the target hosted agent") + } + } else if modelType != agent_api.VoiceModelTypeManaged && modelType != agent_api.VoiceModelTypeSelfDeployed { return nil, fmt.Errorf( - "model_type '%s' is not supported; use '%s' or '%s'", - voiceAgent.ModelType, VoiceModelTypeManaged, VoiceModelTypeSelfDeployed) + "model_type '%s' is not supported; use '%s', '%s', or '%s'", + voiceAgent.ModelType, VoiceModelTypeManaged, VoiceModelTypeSelfDeployed, VoiceModelTypeHostedAgent) + } + if errors := validateVoiceAgentAdvancedConfig(voiceAgent); len(errors) > 0 { + return nil, fmt.Errorf("invalid prompt-voice configuration: %s", strings.Join(errors, "; ")) } - instructions := defaultVoiceInstructions - if voiceAgent.Instructions != nil && *voiceAgent.Instructions != "" { + modelID := "" + if voiceAgent.Model != nil { + modelID = strings.TrimSpace(voiceAgent.Model.Id) + } + if !hostedAgent && modelID == "" { + return nil, fmt.Errorf("model.id is required for a prompt-voice agent") + } + + instructions := "" + if !hostedAgent { + instructions = defaultVoiceInstructions + } + if !hostedAgent && voiceAgent.Instructions != nil && *voiceAgent.Instructions != "" { instructions = *voiceAgent.Instructions } @@ -640,36 +787,77 @@ func createVoiceAgentAPIRequest(voiceAgent VoiceAgent) (*agent_api.CreateAgentRe voiceName = *voiceAgent.Voice } - audioFormat := &agent_api.VoiceAudioFormat{ - Type: defaultVoiceAudioType, - Rate: defaultVoiceAudioRate, + inputFormat := defaultVoiceAudioFormat() + outputFormat := defaultVoiceAudioFormat() + turnDetection := mapVoiceTurnDetection(nil) + transcription := mapVoiceTranscription(nil) + var noiseReduction *agent_api.VoiceNoiseReduction + var echoCancellation map[string]any + outputVoice := buildVoiceConfig(voiceName) + var outputSpeed *float64 + if voiceAgent.Audio != nil { + if voiceAgent.Audio.Input != nil { + inputFormat = mapVoiceAudioFormat(voiceAgent.Audio.Input.Format, inputFormat) + if voiceAgent.Audio.Input.NoiseReduction != nil { + noiseReduction = &agent_api.VoiceNoiseReduction{Type: strings.TrimSpace(voiceAgent.Audio.Input.NoiseReduction.Type)} + } + echoCancellation = voiceAgent.Audio.Input.EchoCancellation + turnDetection = mapVoiceTurnDetection(voiceAgent.Audio.Input.TurnDetection) + transcription = mapVoiceTranscription(voiceAgent.Audio.Input.Transcription) + } + if voiceAgent.Audio.Output != nil { + outputFormat = mapVoiceAudioFormat(voiceAgent.Audio.Output.Format, outputFormat) + outputVoice = mapVoiceConfig(voiceAgent.Audio.Output.Voice, voiceName) + outputSpeed = voiceAgent.Audio.Output.Speed + } + } + + outputModalities := []string{"audio"} + if len(voiceAgent.OutputModalities) > 0 { + outputModalities = voiceAgent.OutputModalities } input := &agent_api.VoiceInputConfig{ - Format: audioFormat, - TurnDetection: &agent_api.VoiceTurnDetection{Type: defaultVoiceTurnDetectionType}, - Transcription: &agent_api.VoiceTranscription{Model: defaultVoiceInputTranscriptionModel}, + Format: inputFormat, + NoiseReduction: noiseReduction, + EchoCancellation: echoCancellation, + TurnDetection: turnDetection, + Transcription: transcription, } - voiceConfig := buildVoiceConfig(voiceName) voiceDef := agent_api.VoiceAgentDefinition{ AgentDefinition: agent_api.AgentDefinition{ // Translate authoring kind prompt-voice -> service kind voice. Kind: agent_api.AgentKindVoice, }, - ModelType: modelType, - Model: modelID, - Instructions: instructions, + ModelType: modelType, + Model: modelID, + TargetAgent: target, + Instructions: instructions, + StructuredInputs: mapVoiceStructuredInputs(voiceAgent.StructuredInputs), Audio: &agent_api.VoiceAudioConfig{ Input: input, Output: &agent_api.VoiceOutputConfig{ - Format: audioFormat, - Voice: voiceConfig.Name, - VoiceType: voiceWireType(voiceConfig), - VoiceLocale: voiceWireLocale(voiceConfig), + Format: outputFormat, + Voice: outputVoice.Name, + VoiceType: voiceWireType(outputVoice), + VoiceLocale: voiceWireLocale(outputVoice), + Style: outputVoice.Style, + Pitch: outputVoice.Pitch, + Rate: outputVoice.Rate, + Volume: outputVoice.Volume, + Speed: outputSpeed, }, }, - OutputModalities: []string{"audio"}, - Store: voiceAgent.Store, + OutputModalities: outputModalities, + Store: voiceAgent.Store, + Tools: voiceAgent.Tools, + Avatar: voiceAgent.Avatar, + Greeting: voiceAgent.Greeting, + Handoff: voiceAgent.Handoff, + ToolChoice: voiceAgent.ToolChoice, + ParallelToolCalls: voiceAgent.ParallelToolCalls, + MaxOutputTokens: voiceAgent.MaxOutputTokens, + Include: voiceAgent.Include, } return createAgentAPIRequest(voiceAgent.AgentDefinition, voiceDef, nil, nil) diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go index 4b603303bfb..3533cce9915 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go @@ -5,6 +5,7 @@ package agent_yaml import ( "encoding/json" + "strings" "testing" "azureaiagent/internal/pkg/agents/agent_api" @@ -118,7 +119,8 @@ func TestCreateVoiceAgentAPIRequest_Defaults(t *testing.T) { t.Fatalf("Audio pipeline not populated: %+v", def.Audio) } in := def.Audio.Input - if in.Format == nil || in.Format.Type != defaultVoiceAudioType || in.Format.Rate != defaultVoiceAudioRate { + if in.Format == nil || in.Format.Type != defaultVoiceAudioType || in.Format.Rate == nil || + *in.Format.Rate != defaultVoiceAudioRate { t.Errorf("input format = %+v", in.Format) } if in.TurnDetection == nil || in.TurnDetection.Type != defaultVoiceTurnDetectionType { @@ -128,7 +130,8 @@ func TestCreateVoiceAgentAPIRequest_Defaults(t *testing.T) { t.Errorf("transcription = %+v", in.Transcription) } out := def.Audio.Output - if out.Format == nil || out.Format.Type != defaultVoiceAudioType || out.Format.Rate != defaultVoiceAudioRate { + if out.Format == nil || out.Format.Type != defaultVoiceAudioType || out.Format.Rate == nil || + *out.Format.Rate != defaultVoiceAudioRate { t.Errorf("output format = %+v", out.Format) } // Default voice is the DragonHD Azure Neural voice in the flat unified shape. @@ -223,6 +226,26 @@ func TestCreateVoiceAgentAPIRequest_UsesAzureVoiceLocale(t *testing.T) { } } +func TestCreateVoiceAgentAPIRequest_PrefersExplicitAzureVoiceLocale(t *testing.T) { + t.Parallel() + locale := "fr-FR" + agent := VoiceAgent{ + AgentDefinition: AgentDefinition{Kind: AgentKindPromptVoice, Name: "voice"}, + Model: &Model{Id: "gpt-realtime"}, + Audio: &VoiceAudio{Output: &VoiceAudioOutput{Voice: &VoiceConfig{ + Type: "azure_standard", Name: "en-US-AvaNeural", Locale: &locale, + }}}, + } + req, err := CreateVoiceAgentAPIRequest(agent) + if err != nil { + t.Fatal(err) + } + def := req.Definition.(agent_api.VoiceAgentDefinition) + if def.Audio.Output.VoiceLocale != locale { + t.Fatalf("VoiceLocale = %q, want explicit %q", def.Audio.Output.VoiceLocale, locale) + } +} + func TestCreateVoiceAgentAPIRequest_UsesAzureVoiceLocaleVariants(t *testing.T) { t.Parallel() tests := []struct { @@ -260,6 +283,85 @@ func TestCreateVoiceAgentAPIRequest_UsesAzureVoiceLocaleVariants(t *testing.T) { } } +func TestCreateVoiceAgentAPIRequest_PrefersExplicitVoiceLocale(t *testing.T) { + t.Parallel() + voiceLocale := "fr-FR" + agent := VoiceAgent{ + AgentDefinition: AgentDefinition{Kind: AgentKindPromptVoice, Name: "voice-agent"}, + Model: &Model{Id: "gpt-realtime"}, + Audio: &VoiceAudio{Output: &VoiceAudioOutput{Voice: &VoiceConfig{ + Type: "azure_standard", Name: "en-US-AvaNeural", Locale: &voiceLocale, + }}}, + } + + req, err := CreateVoiceAgentAPIRequest(agent) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + def := req.Definition.(agent_api.VoiceAgentDefinition) + if def.Audio.Output.VoiceLocale != voiceLocale { + t.Errorf("VoiceLocale = %q, want explicit %q", def.Audio.Output.VoiceLocale, voiceLocale) + } +} + +func TestCreateVoiceAgentAPIRequest_ExplicitOpenAIVoiceLowercasesName(t *testing.T) { + t.Parallel() + agent := VoiceAgent{ + AgentDefinition: AgentDefinition{Kind: AgentKindPromptVoice, Name: "voice-agent"}, + Model: &Model{Id: "gpt-realtime"}, + Audio: &VoiceAudio{Output: &VoiceAudioOutput{Voice: &VoiceConfig{ + Type: "openai", Name: "Shimmer", + }}}, + } + + req, err := CreateVoiceAgentAPIRequest(agent) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + def := req.Definition.(agent_api.VoiceAgentDefinition) + if def.Audio.Output.Voice != "shimmer" { + t.Errorf("Voice = %q, want shimmer", def.Audio.Output.Voice) + } +} + +func TestCreateVoiceAgentAPIRequest_RejectsInvalidAdvancedConfig(t *testing.T) { + t.Parallel() + parallelToolCalls := true + agent := VoiceAgent{ + AgentDefinition: AgentDefinition{Kind: AgentKindPromptVoice, Name: "voice-agent"}, + Model: &Model{Id: "gpt-realtime"}, + ParallelToolCalls: ¶llelToolCalls, + } + _, err := CreateVoiceAgentAPIRequest(agent) + if err == nil || !strings.Contains(err.Error(), "parallel_tool_calls is not currently supported") { + t.Fatalf("expected parallel_tool_calls validation error, got: %v", err) + } +} + +func TestCreateVoiceAgentAPIRequest_DoesNotInheritPcmRateForG711Formats(t *testing.T) { + t.Parallel() + agent := VoiceAgent{ + AgentDefinition: AgentDefinition{Kind: AgentKindPromptVoice, Name: "voice-agent"}, + Model: &Model{Id: "gpt-realtime"}, + Audio: &VoiceAudio{ + Input: &VoiceAudioInput{Format: &VoiceAudioFormat{Type: "audio/pcmu"}}, + Output: &VoiceAudioOutput{Format: &VoiceAudioFormat{Type: "audio/pcma"}}, + }, + } + + req, err := CreateVoiceAgentAPIRequest(agent) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + def := req.Definition.(agent_api.VoiceAgentDefinition) + if def.Audio.Input.Format.Rate != nil { + t.Fatalf("input G.711 rate = %v, want nil", *def.Audio.Input.Format.Rate) + } + if def.Audio.Output.Format.Rate != nil { + t.Fatalf("output G.711 rate = %v, want nil", *def.Audio.Output.Format.Rate) + } +} + func TestCreateVoiceAgentAPIRequest_MarshalServiceWireShape(t *testing.T) { t.Parallel() voice := "en-US-Ava:DragonHDLatestNeural" @@ -309,6 +411,98 @@ func TestCreateVoiceAgentAPIRequest_MarshalServiceWireShape(t *testing.T) { } } +func TestCreateVoiceAgentAPIRequest_AdvancedSettingsWireShape(t *testing.T) { + t.Parallel() + inRate := 16000 + outRate := 24000 + threshold := 0.6 + speechDurationMs := 120 + createResponse := true + removeFillerWords := true + interruptResponse := true + autoTruncate := true + speed := 1.1 + style := "cheerful" + pitch := "+0Hz" + rate := "+0%" + volume := "+0%" + agent := VoiceAgent{ + AgentDefinition: AgentDefinition{Kind: AgentKindPromptVoice, Name: "voice-advanced"}, + Model: &Model{Id: "gpt-realtime"}, + Instructions: new("You are {{persona}}, a concise voice assistant."), + StructuredInputs: map[string]any{ + "persona": map[string]any{"description": "Assistant persona", "defaultValue": "Ada"}, + }, + Audio: &VoiceAudio{ + Input: &VoiceAudioInput{ + Format: &VoiceAudioFormat{Type: "audio/pcmu", Rate: &inRate}, + NoiseReduction: &VoiceNoiseReduction{Type: "near_field"}, + EchoCancellation: map[string]any{"type": "server_echo_cancellation", "channels": 1}, + TurnDetection: &VoiceTurnDetection{ + Type: "azure_semantic_vad", + Threshold: &threshold, + SpeechDurationMs: &speechDurationMs, + CreateResponse: &createResponse, + RemoveFillerWords: &removeFillerWords, + InterruptResponse: &interruptResponse, + Languages: []string{"en-US"}, + AutoTruncate: &autoTruncate, + }, + Transcription: &VoiceTranscription{Model: "azure-speech", Language: new("en-US"), Prompt: new("Contoso terms")}, + }, + Output: &VoiceAudioOutput{ + Format: &VoiceAudioFormat{Type: "audio/pcm", Rate: &outRate}, + Voice: &VoiceConfig{ + Type: "azure_standard", Name: "en-US-AvaNeural", Style: &style, + Pitch: &pitch, Rate: &rate, Locale: new("en-US"), Volume: &volume, + }, + Speed: &speed, + }, + }, + OutputModalities: []string{"audio", "text"}, + Tools: []map[string]any{{"type": "system", "name": "end_conversation"}}, + Avatar: map[string]any{"type": "video_avatar", "character": "lisa", "output_protocol": "webrtc"}, + Greeting: map[string]any{"type": "template", "text": "Hello {{persona}}"}, + ToolChoice: "auto", + MaxOutputTokens: "inf", + Include: []string{"item.input_audio_transcription.phrases"}, + } + + req, err := CreateVoiceAgentAPIRequest(agent) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + payload, err := json.Marshal(req) + if err != nil { + t.Fatalf("marshal request: %v", err) + } + + var wire map[string]any + if err := json.Unmarshal(payload, &wire); err != nil { + t.Fatalf("unmarshal payload: %v", err) + } + def := wire["definition"].(map[string]any) + input := def["audio"].(map[string]any)["input"].(map[string]any) + output := def["audio"].(map[string]any)["output"].(map[string]any) + structured := def["structured_inputs"].(map[string]any)["persona"].(map[string]any) + + if structured["default_value"] != "Ada" || structured["defaultValue"] != nil { + t.Fatalf("structured input default was not mapped to wire shape: %#v", structured) + } + if output["voice"] != "en-US-AvaNeural" || output["voice_type"] != "azure-standard" || output["style"] != style { + t.Fatalf("output voice flat shape not mapped: %#v", output) + } + if input["echo_cancellation"].(map[string]any)["type"] != "server_echo_cancellation" { + t.Fatalf("echo cancellation not mapped: %#v", input["echo_cancellation"]) + } + if def["tool_choice"] != "auto" || def["max_output_tokens"] != "inf" { + t.Fatalf("response options not mapped: %#v", def) + } + if len(def["tools"].([]any)) != 1 || def["avatar"].(map[string]any)["character"] != "lisa" { + t.Fatalf("tools/avatar not mapped: %#v", def) + } +} + // TestCreateVoiceAgentAPIRequest_ExplicitManaged verifies that explicitly // setting model_type: managed is accepted (idempotent with the default). func TestCreateVoiceAgentAPIRequest_ExplicitManaged(t *testing.T) { @@ -400,3 +594,52 @@ func TestCreateVoiceAgentAPIRequest_InvalidModelType(t *testing.T) { t.Error("expected error for unsupported model_type") } } + +func TestCreateHostedVoiceAgentAPIRequest(t *testing.T) { + t.Parallel() + store := false + agent := VoiceAgent{ + AgentDefinition: AgentDefinition{Kind: AgentKindPromptVoice, Name: "voice-wrapper"}, + ModelType: VoiceModelTypeHostedAgent, + TargetAgent: &VoiceTargetAgent{Service: "voice-target", Version: "deployed"}, + Store: &store, + } + req, err := CreateHostedVoiceAgentAPIRequest(agent, agent_api.VoiceTargetAgentReference{ + Name: "deployed-target", Version: "7", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + data, err := json.Marshal(req) + if err != nil { + t.Fatal(err) + } + var wire map[string]any + if err := json.Unmarshal(data, &wire); err != nil { + t.Fatal(err) + } + definition := wire["definition"].(map[string]any) + if definition["model_type"] != "hosted_agent" { + t.Fatalf("model_type = %#v", definition["model_type"]) + } + target := definition["target_agent"].(map[string]any) + if target["name"] != "deployed-target" || target["version"] != "7" { + t.Fatalf("target_agent = %#v", target) + } + for _, field := range []string{"model", "instructions", "structured_inputs", "tools", "tool_choice", "handoff"} { + if _, exists := definition[field]; exists { + t.Fatalf("strict hosted wrapper contains %s: %s", field, data) + } + } +} + +func TestCreateHostedVoiceAgentAPIRequestRequiresResolvedTarget(t *testing.T) { + t.Parallel() + agent := VoiceAgent{ + AgentDefinition: AgentDefinition{Kind: AgentKindPromptVoice, Name: "voice-wrapper"}, + ModelType: VoiceModelTypeHostedAgent, + } + if _, err := CreateHostedVoiceAgentAPIRequest(agent, agent_api.VoiceTargetAgentReference{}); err == nil { + t.Fatal("expected missing resolved target error") + } +} diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go index 996eae8a561..6e7bf784790 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go @@ -450,16 +450,36 @@ func ValidateAgentDefinition(templateBytes []byte) error { case AgentKindPromptVoice: var agent VoiceAgent if err := yaml.Unmarshal(templateBytes, &agent); err == nil { - if agent.Model == nil || strings.TrimSpace(agent.Model.Id) == "" { - errors = append(errors, "template.model.id is required for a prompt-voice agent") + if agent.ModelType == VoiceModelTypeHostedAgent { + if agent.TargetAgent == nil || strings.TrimSpace(agent.TargetAgent.Service) == "" { + errors = append(errors, "template.target_agent.service is required when model_type is 'hosted_agent'") + } + if agent.TargetAgent != nil && agent.TargetAgent.Version != "" && agent.TargetAgent.Version != "deployed" { + errors = append(errors, "template.target_agent.version must be 'deployed' when specified") + } + if agent.Model != nil { + errors = append(errors, "template.model is not allowed when model_type is 'hosted_agent'") + } + if agent.Instructions != nil || len(agent.StructuredInputs) > 0 || len(agent.Tools) > 0 || + agent.ToolChoice != nil || agent.ParallelToolCalls != nil || agent.MaxOutputTokens != nil || + len(agent.Include) > 0 || len(agent.Handoff) > 0 { + errors = append(errors, "instructions, structured_inputs, tools, tool_choice, parallel_tool_calls, max_output_tokens, include, and handoff belong to the target hosted agent") + } + } else { + if agent.Model == nil || strings.TrimSpace(agent.Model.Id) == "" { + errors = append(errors, "template.model.id is required for a prompt-voice agent") + } + if agent.TargetAgent != nil { + errors = append(errors, "template.target_agent is only valid when model_type is 'hosted_agent'") + } } - if agent.ModelType != "" && - agent.ModelType != VoiceModelTypeManaged && - agent.ModelType != VoiceModelTypeSelfDeployed { + if agent.ModelType != "" && agent.ModelType != VoiceModelTypeManaged && + agent.ModelType != VoiceModelTypeSelfDeployed && agent.ModelType != VoiceModelTypeHostedAgent { errors = append(errors, fmt.Sprintf( - "template.model_type '%s' is not supported; use '%s' or '%s'", - agent.ModelType, VoiceModelTypeManaged, VoiceModelTypeSelfDeployed)) + "template.model_type '%s' is not supported; use '%s', '%s', or '%s'", + agent.ModelType, VoiceModelTypeManaged, VoiceModelTypeSelfDeployed, VoiceModelTypeHostedAgent)) } + errors = append(errors, validateVoiceAgentAdvancedConfig(agent)...) } else { errors = append(errors, fmt.Sprintf("failed to unmarshal to VoiceAgent: %v", err)) } @@ -479,6 +499,98 @@ func ValidateAgentDefinition(templateBytes []byte) error { return nil } +func validateVoiceAgentAdvancedConfig(agent VoiceAgent) []string { + var errors []string + for i, modality := range agent.OutputModalities { + if strings.TrimSpace(modality) == "" { + errors = append(errors, fmt.Sprintf("template.output_modalities[%d] must not be blank", i)) + } + } + if agent.ParallelToolCalls != nil { + errors = append(errors, + "template.parallel_tool_calls is not currently supported by the prompt voice runtime; remove it from azure.yaml") + } + + if agent.Audio == nil { + return append(errors, validateVoiceIncludeTranscriptionCompatibility(agent, "")...) + } + transcriptionModel := "" + if agent.Audio.Input != nil { + errors = append(errors, validateVoiceAudioFormat("template.audio.input.format", agent.Audio.Input.Format)...) + if nr := agent.Audio.Input.NoiseReduction; nr != nil && strings.TrimSpace(nr.Type) == "" { + errors = append(errors, "template.audio.input.noise_reduction.type must not be blank") + } + if td := agent.Audio.Input.TurnDetection; td != nil { + if strings.TrimSpace(td.Type) == "" { + errors = append(errors, "template.audio.input.turn_detection.type must not be blank") + } + if td.Threshold != nil && (*td.Threshold <= 0 || *td.Threshold > 1) { + errors = append(errors, "template.audio.input.turn_detection.threshold must be greater than 0 and <= 1") + } + if td.PrefixPaddingMs != nil && *td.PrefixPaddingMs < 0 { + errors = append(errors, "template.audio.input.turn_detection.prefix_padding_ms must be >= 0") + } + if td.SilenceDurationMs != nil && *td.SilenceDurationMs < 0 { + errors = append(errors, "template.audio.input.turn_detection.silence_duration_ms must be >= 0") + } + if td.SpeechDurationMs != nil && *td.SpeechDurationMs < 0 { + errors = append(errors, "template.audio.input.turn_detection.speech_duration_ms must be >= 0") + } + } + if agent.Audio.Input.Transcription != nil { + transcriptionModel = agent.Audio.Input.Transcription.Model + } + } + if agent.Audio.Output != nil { + errors = append(errors, validateVoiceAudioFormat("template.audio.output.format", agent.Audio.Output.Format)...) + if voice := agent.Audio.Output.Voice; voice != nil { + if strings.TrimSpace(voice.Type) == "" { + errors = append(errors, "template.audio.output.voice.type must not be blank") + } + if strings.TrimSpace(voice.Name) == "" { + errors = append(errors, "template.audio.output.voice.name must not be blank") + } + } + if speed := agent.Audio.Output.Speed; speed != nil && (*speed < 0.25 || *speed > 1.5) { + errors = append(errors, "template.audio.output.speed must be between 0.25 and 1.5") + } + } + return append(errors, validateVoiceIncludeTranscriptionCompatibility(agent, transcriptionModel)...) +} + +func validateVoiceIncludeTranscriptionCompatibility(agent VoiceAgent, transcriptionModel string) []string { + if !slices.Contains(agent.Include, "item.input_audio_transcription.phrases") { + return nil + } + model := strings.TrimSpace(transcriptionModel) + if model == "" { + model = defaultVoiceInputTranscriptionModel + } + if model == "azure-speech" || model == "azure-fast-transcription" { + return nil + } + return []string{ + "template.include item.input_audio_transcription.phrases requires template.audio.input.transcription.model to be azure-speech or azure-fast-transcription", + } +} + +func validateVoiceAudioFormat(path string, format *VoiceAudioFormat) []string { + if format == nil { + return nil + } + var errors []string + formatType := strings.TrimSpace(format.Type) + if formatType == "" { + errors = append(errors, path+".type must not be blank") + } else if formatType != "audio/pcm" && formatType != "audio/pcmu" && formatType != "audio/pcma" { + errors = append(errors, path+".type must be 'audio/pcm', 'audio/pcmu', or 'audio/pcma'") + } + if format.Rate != nil && *format.Rate <= 0 { + errors = append(errors, path+".rate must be greater than 0") + } + return errors +} + // Validate that the agent name matches the expected deployable format func ValidateAgentName(name string) error { if name == "" { diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse_voice_test.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse_voice_test.go index 6aba8a368dd..ed103fe7acc 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse_voice_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse_voice_test.go @@ -118,3 +118,177 @@ model_type: unsupported t.Fatalf("expected invalid model_type error, got: %v", err) } } +func TestValidateAgentDefinition_HostedVoiceAccepted(t *testing.T) { + yamlContent := []byte(` +kind: prompt-voice +name: voice-wrapper +model_type: hosted_agent +target_agent: + service: voice-target + version: deployed +`) + if err := ValidateAgentDefinition(yamlContent); err != nil { + t.Fatalf("expected hosted voice definition to be valid, got: %v", err) + } +} + +func TestValidateAgentDefinition_HostedVoiceRequiresTarget(t *testing.T) { + yamlContent := []byte(` +kind: prompt-voice +name: voice-wrapper +model_type: hosted_agent +`) + err := ValidateAgentDefinition(yamlContent) + if err == nil || !strings.Contains(err.Error(), "target_agent.service is required") { + t.Fatalf("expected target agent validation error, got: %v", err) + } +} + +func TestValidateAgentDefinition_HostedVoiceRejectsTargetOwnedFields(t *testing.T) { + yamlContent := []byte(` +kind: prompt-voice +name: voice-wrapper +model_type: hosted_agent +target_agent: + service: voice-target +model: + id: gpt-realtime +instructions: not allowed +`) + err := ValidateAgentDefinition(yamlContent) + if err == nil || !strings.Contains(err.Error(), "belong to the target hosted agent") || + !strings.Contains(err.Error(), "model is not allowed") { + t.Fatalf("expected target-owned field validation errors, got: %v", err) + } +} + +func TestValidateAgentDefinition_PromptVoice_RejectsParallelToolCalls(t *testing.T) { + yamlContent := []byte(` +kind: prompt-voice +name: voice-agent +model: + id: gpt-realtime +parallel_tool_calls: true +`) + err := ValidateAgentDefinition(yamlContent) + if err == nil || !strings.Contains(err.Error(), "parallel_tool_calls is not currently supported") { + t.Fatalf("expected parallel_tool_calls validation error, got: %v", err) + } +} + +func TestValidateAgentDefinition_PromptVoice_RejectsZeroTurnDetectionThreshold(t *testing.T) { + yamlContent := []byte(` +kind: prompt-voice +name: voice-agent +model: + id: gpt-realtime +audio: + input: + turn_detection: + type: azure_semantic_vad + threshold: 0 +`) + err := ValidateAgentDefinition(yamlContent) + if err == nil || !strings.Contains(err.Error(), "threshold must be greater than 0") { + t.Fatalf("expected threshold validation error, got: %v", err) + } +} + +func TestValidateAgentDefinition_PromptVoice_InvalidIncludeTranscriptionModel(t *testing.T) { + yamlContent := []byte(` +kind: prompt-voice +name: voice-agent +model: + id: gpt-realtime +audio: + input: + transcription: + model: whisper-1 +include: + - item.input_audio_transcription.phrases +`) + err := ValidateAgentDefinition(yamlContent) + if err == nil { + t.Fatal("expected include/transcription validation error") + } + if !strings.Contains(err.Error(), "azure-speech") || !strings.Contains(err.Error(), "azure-fast-transcription") { + t.Fatalf("expected transcription model guidance in error, got: %v", err) + } +} +func TestValidateAgentDefinition_PromptVoice_AdvancedValidationBoundaries(t *testing.T) { + tests := []struct { + name string + yaml string + want string + }{ + { + name: "unsupported format", + yaml: `audio: + input: + format: + type: audio/opus`, + want: "audio/pcm", + }, + { + name: "invalid rate", + yaml: `audio: + input: + format: + type: audio/pcm + rate: 0`, + want: "rate must be greater than 0", + }, + { + name: "negative duration", + yaml: `audio: + input: + turn_detection: + type: azure_semantic_vad + speech_duration_ms: -1`, + want: "speech_duration_ms must be >= 0", + }, + { + name: "blank voice name", + yaml: `audio: + output: + voice: + type: azure_standard + name: ""`, + want: "voice.name must not be blank", + }, + { + name: "invalid speed", + yaml: `audio: + output: + speed: 2`, + want: "speed must be between 0.25 and 1.5", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + yamlContent := []byte("kind: prompt-voice\nname: voice-agent\nmodel:\n id: gpt-realtime\n" + tt.yaml + "\n") + err := ValidateAgentDefinition(yamlContent) + if err == nil || !strings.Contains(err.Error(), tt.want) { + t.Fatalf("expected %q validation error, got: %v", tt.want, err) + } + }) + } +} + +func TestValidateAgentDefinition_PromptVoice_ValidIncludeTranscriptionModel(t *testing.T) { + yamlContent := []byte(` +kind: prompt-voice +name: voice-agent +model: + id: gpt-realtime +audio: + input: + transcription: + model: azure-speech +include: + - item.input_audio_transcription.phrases +`) + if err := ValidateAgentDefinition(yamlContent); err != nil { + t.Fatalf("expected azure-speech include config to be valid, got: %v", err) + } +} diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/yaml.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/yaml.go index 46fffca9e6a..a27855a86a8 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/yaml.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/yaml.go @@ -30,8 +30,17 @@ type VoiceModelType string const ( VoiceModelTypeManaged VoiceModelType = "managed" VoiceModelTypeSelfDeployed VoiceModelType = "self_deployed" + VoiceModelTypeHostedAgent VoiceModelType = "hosted_agent" ) +// VoiceTargetAgent identifies the hosted agent service that supplies the +// conversation logic for a hosted voice wrapper. Service is an azure.yaml +// service name; azd resolves it to the deployed Foundry agent name and version. +type VoiceTargetAgent struct { + Service string `json:"service" yaml:"service"` + Version string `json:"version,omitempty" yaml:"version,omitempty"` +} + // IsValidAgentKind checks if the provided AgentKind is valid func IsValidAgentKind(kind AgentKind) bool { return slices.Contains(ValidAgentKinds(), kind) @@ -204,19 +213,111 @@ type Workflow struct { // map layer so authors don't have to specify it. type VoiceAgent struct { AgentDefinition `json:",inline" yaml:",inline"` + Policies []Policy `json:"policies,omitempty" yaml:"policies,omitempty"` // ModelType selects managed vs self_deployed (BYOM). Optional; defaults to managed. ModelType VoiceModelType `json:"modelType,omitempty" yaml:"model_type,omitempty"` // Model names the speech-to-speech model (e.g. "gpt-realtime"). Reuses the // shared Model struct; only Id is required for voice. Model *Model `json:"model,omitempty" yaml:"model,omitempty"` + // TargetAgent references the hosted agent service used when model_type is hosted_agent. + TargetAgent *VoiceTargetAgent `json:"targetAgent,omitempty" yaml:"target_agent,omitempty"` // Instructions is the system prompt for the voice assistant. Instructions *string `json:"instructions,omitempty" yaml:"instructions,omitempty"` // Voice is the output voice name (e.g. "en-US-Ava:DragonHDLatestNeural" for // an Azure Neural voice, or "alloy" for an OpenAI realtime voice). Voice *string `json:"voice,omitempty" yaml:"voice,omitempty"` + // StructuredInputs declares template inputs used by voice instructions and greeting. + StructuredInputs map[string]any `json:"structuredInputs,omitempty" yaml:"structured_inputs,omitempty"` + // Audio customizes the input and output voice pipeline. Missing fields keep azd defaults. + Audio *VoiceAudio `json:"audio,omitempty" yaml:"audio,omitempty"` + // OutputModalities declares response modalities such as audio, text, animation, or avatar. + OutputModalities []string `json:"outputModalities,omitempty" yaml:"output_modalities,omitempty"` // Store toggles server-side logging (transcript + per-turn audio). Optional; // the service defaults to false when omitted. Store *bool `json:"store,omitempty" yaml:"store,omitempty"` + // Tools are passed through to the prompt voice service. Supported direct tool + // types include function, mcp, system, and toolbox. + Tools []map[string]any `json:"tools,omitempty" yaml:"tools,omitempty"` + // Avatar customizes voice avatar output for services that support it. + Avatar map[string]any `json:"avatar,omitempty" yaml:"avatar,omitempty"` + // Greeting configures initial greeting behavior for services that support it. + Greeting map[string]any `json:"greeting,omitempty" yaml:"greeting,omitempty"` + // Handoff configures voice handoff behavior for services that support it. + Handoff map[string]any `json:"handoff,omitempty" yaml:"handoff,omitempty"` + // ToolChoice configures service tool choice behavior, such as auto/none/required. + ToolChoice any `json:"toolChoice,omitempty" yaml:"tool_choice,omitempty"` + // ParallelToolCalls toggles parallel tool calls. + ParallelToolCalls *bool `json:"parallelToolCalls,omitempty" yaml:"parallel_tool_calls,omitempty"` + // MaxOutputTokens limits response output tokens. Use an integer or service-supported string such as "inf". + MaxOutputTokens any `json:"maxOutputTokens,omitempty" yaml:"max_output_tokens,omitempty"` + // Include requests additional service response fields. + Include []string `json:"include,omitempty" yaml:"include,omitempty"` +} + +// VoiceAudio bundles optional prompt voice input/output audio overrides. +type VoiceAudio struct { + Input *VoiceAudioInput `json:"input,omitempty" yaml:"input,omitempty"` + Output *VoiceAudioOutput `json:"output,omitempty" yaml:"output,omitempty"` +} + +// VoiceAudioInput customizes caller-to-agent audio. +type VoiceAudioInput struct { + Format *VoiceAudioFormat `json:"format,omitempty" yaml:"format,omitempty"` + NoiseReduction *VoiceNoiseReduction `json:"noiseReduction,omitempty" yaml:"noise_reduction,omitempty"` + EchoCancellation map[string]any `json:"echoCancellation,omitempty" yaml:"echo_cancellation,omitempty"` + TurnDetection *VoiceTurnDetection `json:"turnDetection,omitempty" yaml:"turn_detection,omitempty"` + Transcription *VoiceTranscription `json:"transcription,omitempty" yaml:"transcription,omitempty"` +} + +// VoiceAudioOutput customizes agent-to-caller audio. +type VoiceAudioOutput struct { + Format *VoiceAudioFormat `json:"format,omitempty" yaml:"format,omitempty"` + Voice *VoiceConfig `json:"voice,omitempty" yaml:"voice,omitempty"` + Speed *float64 `json:"speed,omitempty" yaml:"speed,omitempty"` +} + +// VoiceAudioFormat describes an audio stream format. +type VoiceAudioFormat struct { + Type string `json:"type" yaml:"type"` + Rate *int `json:"rate,omitempty" yaml:"rate,omitempty"` +} + +// VoiceNoiseReduction configures input audio noise reduction. +type VoiceNoiseReduction struct { + Type string `json:"type" yaml:"type"` +} + +// VoiceTurnDetection configures server-side turn detection. +type VoiceTurnDetection struct { + Type string `json:"type" yaml:"type"` + Threshold *float64 `json:"threshold,omitempty" yaml:"threshold,omitempty"` + PrefixPaddingMs *int `json:"prefixPaddingMs,omitempty" yaml:"prefix_padding_ms,omitempty"` + SilenceDurationMs *int `json:"silenceDurationMs,omitempty" yaml:"silence_duration_ms,omitempty"` + CreateResponse *bool `json:"createResponse,omitempty" yaml:"create_response,omitempty"` + Eagerness *string `json:"eagerness,omitempty" yaml:"eagerness,omitempty"` + SpeechDurationMs *int `json:"speechDurationMs,omitempty" yaml:"speech_duration_ms,omitempty"` + RemoveFillerWords *bool `json:"removeFillerWords,omitempty" yaml:"remove_filler_words,omitempty"` + InterruptResponse *bool `json:"interruptResponse,omitempty" yaml:"interrupt_response,omitempty"` + Languages []string `json:"languages,omitempty" yaml:"languages,omitempty"` + AutoTruncate *bool `json:"autoTruncate,omitempty" yaml:"auto_truncate,omitempty"` +} + +// VoiceTranscription configures input transcription. +type VoiceTranscription struct { + Model string `json:"model,omitempty" yaml:"model,omitempty"` + Language *string `json:"language,omitempty" yaml:"language,omitempty"` + Prompt *string `json:"prompt,omitempty" yaml:"prompt,omitempty"` +} + +// VoiceConfig selects the output voice. +type VoiceConfig struct { + Type string `json:"type" yaml:"type"` + Name string `json:"name" yaml:"name"` + Style *string `json:"style,omitempty" yaml:"style,omitempty"` + Pitch *string `json:"pitch,omitempty" yaml:"pitch,omitempty"` + Rate *string `json:"rate,omitempty" yaml:"rate,omitempty"` + Locale *string `json:"locale,omitempty" yaml:"locale,omitempty"` + Volume *string `json:"volume,omitempty" yaml:"volume,omitempty"` } // ContainerResources represents the resource allocation for a containerized agent. diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/agent_definition.go b/cli/azd/extensions/azure.ai.agents/internal/project/agent_definition.go index 26b5a06f874..642a46e5d6a 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/agent_definition.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/agent_definition.go @@ -138,35 +138,73 @@ type AgentDefinitionInline struct { // Voice-agent fields (kind: prompt-voice). All omitempty so container/ // workflow entries are byte-for-byte unchanged. - ModelType agent_yaml.VoiceModelType `json:"modelType,omitempty"` - Model *agent_yaml.Model `json:"model,omitempty"` - Instructions *string `json:"instructions,omitempty"` - Voice *string `json:"voice,omitempty"` - Store *bool `json:"store,omitempty"` + ModelType agent_yaml.VoiceModelType `json:"modelType,omitempty"` + Model *agent_yaml.Model `json:"model,omitempty"` + TargetAgent *agent_yaml.VoiceTargetAgent `json:"targetAgent,omitempty"` + Instructions *string `json:"instructions,omitempty"` + Voice *string `json:"voice,omitempty"` + StructuredInputs map[string]any `json:"structuredInputs,omitempty"` + Audio *agent_yaml.VoiceAudio `json:"audio,omitempty"` + OutputModalities []string `json:"outputModalities,omitempty"` + Store *bool `json:"store,omitempty"` + Tools []map[string]any `json:"tools,omitempty"` + Avatar map[string]any `json:"avatar,omitempty"` + Greeting map[string]any `json:"greeting,omitempty"` + Handoff map[string]any `json:"handoff,omitempty"` + ToolChoice any `json:"toolChoice,omitempty"` + ParallelToolCalls *bool `json:"parallelToolCalls,omitempty"` + MaxOutputTokens any `json:"maxOutputTokens,omitempty"` + Include []string `json:"include,omitempty"` } // voiceAgentDefinitionToInline projects a VoiceAgent into the inline definition // written to azure.yaml. Voice agents carry no container/image/code config. func voiceAgentDefinitionToInline(va agent_yaml.VoiceAgent) AgentDefinitionInline { return AgentDefinitionInline{ - AgentDefinition: va.AgentDefinition, - ModelType: va.ModelType, - Model: va.Model, - Instructions: va.Instructions, - Voice: va.Voice, - Store: va.Store, + AgentDefinition: va.AgentDefinition, + ModelType: va.ModelType, + Model: va.Model, + TargetAgent: va.TargetAgent, + Instructions: va.Instructions, + Voice: va.Voice, + StructuredInputs: va.StructuredInputs, + Audio: va.Audio, + OutputModalities: va.OutputModalities, + Store: va.Store, + Tools: va.Tools, + Avatar: va.Avatar, + Greeting: va.Greeting, + Handoff: va.Handoff, + ToolChoice: va.ToolChoice, + ParallelToolCalls: va.ParallelToolCalls, + MaxOutputTokens: va.MaxOutputTokens, + Include: va.Include, + Policies: va.Policies, } } // toVoiceAgent rebuilds an agent_yaml.VoiceAgent from the inline definition. func (d AgentDefinitionInline) toVoiceAgent() agent_yaml.VoiceAgent { return agent_yaml.VoiceAgent{ - AgentDefinition: d.AgentDefinition, - ModelType: d.ModelType, - Model: d.Model, - Instructions: d.Instructions, - Voice: d.Voice, - Store: d.Store, + AgentDefinition: d.AgentDefinition, + ModelType: d.ModelType, + Model: d.Model, + TargetAgent: d.TargetAgent, + Instructions: d.Instructions, + Voice: d.Voice, + StructuredInputs: d.StructuredInputs, + Audio: d.Audio, + OutputModalities: d.OutputModalities, + Store: d.Store, + Tools: d.Tools, + Avatar: d.Avatar, + Greeting: d.Greeting, + Handoff: d.Handoff, + ToolChoice: d.ToolChoice, + ParallelToolCalls: d.ParallelToolCalls, + MaxOutputTokens: d.MaxOutputTokens, + Include: d.Include, + Policies: d.Policies, } } @@ -749,7 +787,11 @@ func agentDefinitionFromStruct( } if inline.Kind != agent_yaml.AgentKindHosted { - if err := validateAgentServiceDefinition(s.AsMap()); err != nil { + definition := any(s.AsMap()) + if inline.Kind == agent_yaml.AgentKindPromptVoice { + definition = inline.toVoiceAgent() + } + if err := validateAgentServiceDefinition(definition); err != nil { return agent_yaml.ContainerAgent{}, false, err } return agent_yaml.ContainerAgent{}, false, nil diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/agent_policies_test.go b/cli/azd/extensions/azure.ai.agents/internal/project/agent_policies_test.go index 0fb7423f327..be9fc35d9be 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/agent_policies_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/agent_policies_test.go @@ -374,7 +374,7 @@ func TestAgentPoliciesInvocationsModerationNonHostedInline(t *testing.T) { t.Run(kind, func(t *testing.T) { t.Parallel() - _, _, _, _, err := AgentDefinitionFromService(inlineAgentService(t, map[string]any{ + properties := map[string]any{ "kind": kind, "name": "rai-agent", "policies": []any{ @@ -388,7 +388,11 @@ func TestAgentPoliciesInvocationsModerationNonHostedInline(t *testing.T) { }, }, }, - })) + } + if kind == "prompt-voice" { + properties["model"] = map[string]any{"id": "gpt-realtime"} + } + _, _, _, _, err := AgentDefinitionFromService(inlineAgentService(t, properties)) require.ErrorContains(t, err, "invocationsModeration is only supported for 'hosted' agents") }) } diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/doc_examples_test.go b/cli/azd/extensions/azure.ai.agents/internal/project/doc_examples_test.go index d0260148930..e3240bb53c5 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/doc_examples_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/doc_examples_test.go @@ -548,6 +548,7 @@ func serviceConfigFromDoc(t *testing.T, e docExample, name string, svc map[strin RelativePath: core.RelativePath, Image: core.Image, AdditionalProperties: props, + Uses: core.Uses, } // The deprecated shape nests the agent definition under `config`. diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/foundry_dependencies.go b/cli/azd/extensions/azure.ai.agents/internal/project/foundry_dependencies.go index 6249656fe85..76d23397818 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/foundry_dependencies.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/foundry_dependencies.go @@ -12,6 +12,7 @@ import ( "strings" "azureaiagent/internal/exterrors" + "azureaiagent/internal/pkg/agents/agent_yaml" "azureaiagent/internal/pkg/envkey" "github.com/azure/azure-dev/cli/azd/pkg/azdext" @@ -19,6 +20,13 @@ import ( type dependencyEnabled func(context.Context, string) (bool, error) +type hostedVoiceTarget struct { + ServiceName string + AgentName string + AgentVersion string + ProjectEndpoint string +} + const ( foundryProjectHost = "azure.ai.project" foundryConnectionHost = "azure.ai.connection" @@ -348,6 +356,64 @@ func validateFoundryAgentDependency(service *azdext.ServiceConfig, env map[strin return "" } +func resolveHostedVoiceTarget( + wrapper *azdext.ServiceConfig, + voiceAgentTarget *agent_yaml.VoiceTargetAgent, + services map[string]*azdext.ServiceConfig, + env map[string]string, + projectRoot string, +) (*hostedVoiceTarget, error) { + if voiceAgentTarget == nil || strings.TrimSpace(voiceAgentTarget.Service) == "" { + return nil, fmt.Errorf("targetAgent.service is required when modelType is hosted_agent") + } + targetServiceName := strings.TrimSpace(voiceAgentTarget.Service) + if !slices.Contains(wrapper.GetUses(), targetServiceName) { + return nil, fmt.Errorf( + "hosted voice target service %q must be declared in the %q service uses list", + targetServiceName, wrapper.GetName()) + } + targetService, ok := services[targetServiceName] + if !ok { + return nil, fmt.Errorf("hosted voice target service %q was not found in azure.yaml", targetServiceName) + } + if targetService.GetHost() != foundryAgentHost { + return nil, fmt.Errorf( + "hosted voice target service %q must use host %q, got %q", + targetServiceName, foundryAgentHost, targetService.GetHost()) + } + _, isHosted, _, err := LoadAgentDefinition(targetService, projectRoot) + if err != nil { + return nil, fmt.Errorf("loading hosted voice target service %q: %w", targetServiceName, err) + } + if !isHosted { + return nil, fmt.Errorf("hosted voice target service %q must have kind hosted", targetServiceName) + } + + key := normalizeAgentServiceKey(targetServiceName) + name := strings.TrimSpace(env[fmt.Sprintf("AGENT_%s_NAME", key)]) + version := strings.TrimSpace(env[fmt.Sprintf("AGENT_%s_VERSION", key)]) + projectEndpoint := strings.TrimSpace(env[envkey.AgentProjectEndpoint(targetServiceName)]) + baseEndpoint := strings.TrimSpace(env[fmt.Sprintf("AGENT_%s_ENDPOINT", key)]) + if projectEndpoint == "" && endpointBelongsToProject(baseEndpoint, env["FOUNDRY_PROJECT_ENDPOINT"]) { + projectEndpoint = strings.TrimRight(strings.TrimSpace(env["FOUNDRY_PROJECT_ENDPOINT"]), "/") + } + if name == "" || version == "" || projectEndpoint == "" { + return nil, fmt.Errorf( + "hosted voice target service %q is not deployed; run 'azd deploy %s' or 'azd deploy --all'", + targetServiceName, strconv.Quote(targetServiceName)) + } + if !sameProjectEndpoint(projectEndpoint, env["FOUNDRY_PROJECT_ENDPOINT"]) { + return nil, fmt.Errorf("hosted voice target service %q is deployed to a different Foundry project", targetServiceName) + } + + return &hostedVoiceTarget{ + ServiceName: targetServiceName, + AgentName: name, + AgentVersion: version, + ProjectEndpoint: projectEndpoint, + }, nil +} + func endpointBelongsToProject(resourceEndpoint, projectEndpoint string) bool { resourceEndpoint = strings.TrimRight(strings.TrimSpace(resourceEndpoint), "/") projectEndpoint = strings.TrimRight(strings.TrimSpace(projectEndpoint), "/") diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/hosted_voice_target_test.go b/cli/azd/extensions/azure.ai.agents/internal/project/hosted_voice_target_test.go new file mode 100644 index 00000000000..dcd445c29f8 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/internal/project/hosted_voice_target_test.go @@ -0,0 +1,146 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "testing" + + "azureaiagent/internal/pkg/agents/agent_api" + "azureaiagent/internal/pkg/agents/agent_yaml" + "azureaiagent/internal/pkg/envkey" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/require" +) + +func TestResolveHostedVoiceTarget(t *testing.T) { + targetProps, err := AgentDefinitionToServiceProperties(agent_yaml.ContainerAgent{ + AgentDefinition: agent_yaml.AgentDefinition{Kind: agent_yaml.AgentKindHosted, Name: "remote-target"}, + }, nil) + require.NoError(t, err) + target := &azdext.ServiceConfig{ + Name: "voice-target", + Host: foundryAgentHost, + AdditionalProperties: targetProps, + } + wrapper := &azdext.ServiceConfig{ + Name: "voice-wrapper", + Host: foundryAgentHost, + Uses: []string{"voice-target"}, + } + projectEndpoint := "https://account.services.ai.azure.com/api/projects/project" + env := map[string]string{ + "FOUNDRY_PROJECT_ENDPOINT": projectEndpoint, + "AGENT_VOICE_TARGET_NAME": "remote-target", + "AGENT_VOICE_TARGET_VERSION": "4", + envkey.AgentProjectEndpoint("voice-target"): projectEndpoint, + } + + resolved, err := resolveHostedVoiceTarget( + wrapper, + &agent_yaml.VoiceTargetAgent{Service: "voice-target", Version: "deployed"}, + map[string]*azdext.ServiceConfig{"voice-target": target}, + env, + t.TempDir(), + ) + require.NoError(t, err) + require.Equal(t, "remote-target", resolved.AgentName) + require.Equal(t, "4", resolved.AgentVersion) +} + +func TestResolveHostedVoiceTargetRequiresUses(t *testing.T) { + wrapper := &azdext.ServiceConfig{Name: "voice-wrapper", Host: foundryAgentHost} + _, err := resolveHostedVoiceTarget( + wrapper, + &agent_yaml.VoiceTargetAgent{Service: "voice-target"}, + map[string]*azdext.ServiceConfig{}, + map[string]string{}, + t.TempDir(), + ) + require.ErrorContains(t, err, "uses list") +} + +func TestResolveHostedVoiceTargetSupportsLegacyEndpointMarker(t *testing.T) { + targetProps, err := AgentDefinitionToServiceProperties(agent_yaml.ContainerAgent{ + AgentDefinition: agent_yaml.AgentDefinition{Kind: agent_yaml.AgentKindHosted, Name: "remote-target"}, + }, nil) + require.NoError(t, err) + target := &azdext.ServiceConfig{Name: "voice-target", Host: foundryAgentHost, AdditionalProperties: targetProps} + wrapper := &azdext.ServiceConfig{Name: "voice-wrapper", Host: foundryAgentHost, Uses: []string{"voice-target"}} + projectEndpoint := "https://account.services.ai.azure.com/api/projects/project" + resolved, err := resolveHostedVoiceTarget( + wrapper, + &agent_yaml.VoiceTargetAgent{Service: "voice-target"}, + map[string]*azdext.ServiceConfig{"voice-target": target}, + map[string]string{ + "FOUNDRY_PROJECT_ENDPOINT": projectEndpoint, + "AGENT_VOICE_TARGET_NAME": "remote-target", + "AGENT_VOICE_TARGET_VERSION": "4", + "AGENT_VOICE_TARGET_ENDPOINT": projectEndpoint + "/agents/remote-target/versions/4", + }, + t.TempDir(), + ) + require.NoError(t, err) + require.Equal(t, projectEndpoint, resolved.ProjectEndpoint) +} + +func TestResolveHostedVoiceTargetRejectsDifferentProject(t *testing.T) { + targetProps, err := AgentDefinitionToServiceProperties(agent_yaml.ContainerAgent{ + AgentDefinition: agent_yaml.AgentDefinition{Kind: agent_yaml.AgentKindHosted, Name: "remote-target"}, + }, nil) + require.NoError(t, err) + target := &azdext.ServiceConfig{Name: "voice-target", Host: foundryAgentHost, AdditionalProperties: targetProps} + wrapper := &azdext.ServiceConfig{Name: "voice-wrapper", Host: foundryAgentHost, Uses: []string{"voice-target"}} + _, err = resolveHostedVoiceTarget( + wrapper, + &agent_yaml.VoiceTargetAgent{Service: "voice-target"}, + map[string]*azdext.ServiceConfig{"voice-target": target}, + map[string]string{ + "FOUNDRY_PROJECT_ENDPOINT": "https://account.services.ai.azure.com/api/projects/current", + "AGENT_VOICE_TARGET_NAME": "remote-target", + "AGENT_VOICE_TARGET_VERSION": "4", + envkey.AgentProjectEndpoint("voice-target"): "https://account.services.ai.azure.com/api/projects/other", + }, + t.TempDir(), + ) + require.ErrorContains(t, err, "different Foundry project") +} + +func TestValidateHostedVoiceTarget(t *testing.T) { + err := validateHostedVoiceTargetVersion(&agent_api.AgentVersionObject{ + Name: "remote-target", + Version: "4", + Status: "active", + Metadata: map[string]string{ + "voiceLiveCompatible": "true", + "bridgeProtocolVersion": "1.0", + }, + Definition: map[string]any{ + "kind": "hosted", + "protocol_versions": []any{map[string]any{ + "protocol": "invocations_ws", + "version": "1.0.0", + }}, + }, + }) + require.NoError(t, err) +} + +func TestValidateHostedVoiceTargetRejectsIncompatibleProtocol(t *testing.T) { + err := validateHostedVoiceTargetVersion(&agent_api.AgentVersionObject{ + Status: "active", + Metadata: map[string]string{ + "voiceLiveCompatible": "true", + "bridgeProtocolVersion": "1.0", + }, + Definition: map[string]any{ + "kind": "hosted", + "protocol_versions": []any{map[string]any{ + "protocol": "responses", + "version": "2.0.0", + }}, + }, + }) + require.ErrorContains(t, err, "invocations_ws/1.0.0") +} diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go index 59e2b162a40..af838dc3556 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go @@ -10,6 +10,7 @@ import ( "crypto/sha256" "encoding/base64" "encoding/hex" + "encoding/json" "errors" "fmt" "io" @@ -2216,7 +2217,39 @@ func (p *AgentServiceTargetProvider) deployVoiceAgent( ) (*azdext.ServiceDeployResult, error) { progress("Deploying voice agent") - request, err := agent_yaml.CreateVoiceAgentAPIRequest(va) + var request *agent_api.CreateAgentRequest + var hostedTarget *hostedVoiceTarget + var err error + if va.ModelType == agent_yaml.VoiceModelTypeHostedAgent { + if va.TargetAgent != nil && p.dependencyEnabled != nil { + enabled, enabledErr := p.dependencyEnabled(ctx, strings.TrimSpace(va.TargetAgent.Service)) + if enabledErr != nil { + return nil, enabledErr + } + if !enabled { + return nil, exterrors.Dependency( + exterrors.CodeFoundryDependencyNotReady, + fmt.Sprintf("hosted voice target service %q is disabled", va.TargetAgent.Service), + "enable the target hosted agent service or remove the voice wrapper", + ) + } + } + hostedTarget, err = resolveHostedVoiceTarget( + serviceConfig, va.TargetAgent, p.projectServices, azdEnv, p.projectPath, + ) + if err != nil { + return nil, exterrors.Dependency( + exterrors.CodeFoundryDependencyNotReady, + fmt.Sprintf("cannot resolve hosted voice target: %s", err), + "deploy the target hosted agent in the same project, then retry", + ) + } + request, err = agent_yaml.CreateHostedVoiceAgentAPIRequest(va, agent_api.VoiceTargetAgentReference{ + Name: hostedTarget.AgentName, Version: hostedTarget.AgentVersion, + }) + } else { + request, err = agent_yaml.CreateVoiceAgentAPIRequest(va) + } if err != nil { return nil, exterrors.Validation( exterrors.CodeInvalidAgentManifest, @@ -2236,6 +2269,16 @@ func (p *AgentServiceTargetProvider) deployVoiceAgent( } agentClient := agent_api.NewAgentClient(projectEndpoint, p.credential) + if hostedTarget != nil { + progress("Validating hosted voice target") + if err := validateHostedVoiceTarget(ctx, agentClient, hostedTarget); err != nil { + return nil, exterrors.Dependency( + exterrors.CodeFoundryDependencyNotReady, + fmt.Sprintf("hosted voice target is not compatible: %s", err), + "deploy an active Voice Bridge 1.0 hosted agent with invocations_ws/1.0.0, then retry", + ) + } + } serviceKey := p.getServiceKey(serviceConfig.Name) agentObject, deployOp, err := p.deployVoiceAgentRemote( @@ -2263,10 +2306,18 @@ func (p *AgentServiceTargetProvider) deployVoiceAgent( }); setErr != nil { return nil, fmt.Errorf("clearing voice agent environment variable %s: %w", endpointKey, setErr) } + targetName := "" + targetVersion := "" + if hostedTarget != nil { + targetName = hostedTarget.AgentName + targetVersion = hostedTarget.AgentVersion + } for _, envVar := range []struct{ key, value string }{ {fmt.Sprintf("AGENT_%s_NAME", serviceKey), agentObject.Name}, {versionKey, versionValue}, {fmt.Sprintf("AGENT_%s_PROJECT_ENDPOINT", serviceKey), strings.TrimRight(projectEndpoint, "/")}, + {fmt.Sprintf("AGENT_%s_TARGET_NAME", serviceKey), targetName}, + {fmt.Sprintf("AGENT_%s_TARGET_VERSION", serviceKey), targetVersion}, {endpointKey, baseEndpoint}, } { if _, setErr := p.azdClient.Environment().SetValue(ctx, &azdext.SetEnvRequest{ @@ -2292,6 +2343,57 @@ func (p *AgentServiceTargetProvider) deployVoiceAgent( return &azdext.ServiceDeployResult{Artifacts: artifacts}, nil } +func validateHostedVoiceTarget( + ctx context.Context, + agentClient *agent_api.AgentClient, + target *hostedVoiceTarget, +) error { + version, err := agentClient.GetAgentVersion( + ctx, target.AgentName, target.AgentVersion, agent_api.AgentEndpointAPIVersion, + ) + if err != nil { + return fmt.Errorf("getting target %s:%s: %w", target.AgentName, target.AgentVersion, err) + } + return validateHostedVoiceTargetVersion(version) +} + +func validateHostedVoiceTargetVersion(version *agent_api.AgentVersionObject) error { + if version == nil { + return fmt.Errorf("target version response is empty") + } + if version.Status != "active" { + return fmt.Errorf("target %s:%s has status %q, expected active", version.Name, version.Version, version.Status) + } + definitionJSON, err := json.Marshal(version.Definition) + if err != nil { + return fmt.Errorf("reading target definition: %w", err) + } + var definition agent_api.HostedAgentDefinition + if err := json.Unmarshal(definitionJSON, &definition); err != nil { + return fmt.Errorf("reading target definition: %w", err) + } + if definition.Kind != agent_api.AgentKindHosted { + return fmt.Errorf("target kind is %q, expected hosted", definition.Kind) + } + compatibleProtocol := false + for _, protocol := range definition.ProtocolVersions { + if protocol.Protocol == agent_api.AgentProtocolInvocationsWS && protocol.Version == "1.0.0" { + compatibleProtocol = true + break + } + } + if !compatibleProtocol { + return fmt.Errorf("target does not declare invocations_ws/1.0.0") + } + if !strings.EqualFold(strings.TrimSpace(version.Metadata["voiceLiveCompatible"]), "true") { + return fmt.Errorf("target metadata voiceLiveCompatible must be true") + } + if strings.TrimSpace(version.Metadata["bridgeProtocolVersion"]) != "1.0" { + return fmt.Errorf("target metadata bridgeProtocolVersion must be 1.0") + } + return nil +} + func validateVoiceAgentDeployResponse(agentObject *agent_api.AgentObject) error { if agentObject == nil { return fmt.Errorf("malformed voice agent service response: missing agent object") diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go index a65a691a95e..aa5bb4880d4 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go @@ -69,6 +69,33 @@ func TestVoiceAgentInlineServicePropertiesRoundTrip_BYOM(t *testing.T) { require.Equal(t, store, *got.Store) } +func TestVoiceAgentInlineServicePropertiesRoundTrip_HostedAgent(t *testing.T) { + props, err := VoiceAgentDefinitionToServiceProperties(agent_yaml.VoiceAgent{ + AgentDefinition: agent_yaml.AgentDefinition{ + Kind: agent_yaml.AgentKindPromptVoice, + Name: "voice-wrapper", + }, + ModelType: agent_yaml.VoiceModelTypeHostedAgent, + TargetAgent: &agent_yaml.VoiceTargetAgent{ + Service: "voice-target", + Version: "deployed", + }, + }, nil) + require.NoError(t, err) + + svc := &azdext.ServiceConfig{ + Name: "voice-wrapper", + Host: "azure.ai.agent", + AdditionalProperties: props, + } + got, found, err := VoiceAgentFromResolvedService(svc, t.TempDir()) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, agent_yaml.VoiceModelTypeHostedAgent, got.ModelType) + require.Equal(t, "voice-target", got.TargetAgent.Service) + require.Equal(t, "deployed", got.TargetAgent.Version) +} + func TestApplyAgentMetadata(t *testing.T) { tests := []struct { name string diff --git a/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json b/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json index 553e7692fd2..4caecd29b51 100644 --- a/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json +++ b/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json @@ -1,531 +1,704 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Azure AI Agent Service Target Configuration", - "description": "Custom configuration for the Azure AI Agent Service target", - "type": "object", - "properties": { - "container": { - "$ref": "#/definitions/ContainerSettings" - }, - "deployments": { - "type": "array", - "description": "List of model deployments.", - "items": { "$ref": "#/definitions/Deployment" } - }, - "resources": { - "type": "array", - "description": "List of external resources for agent execution.", - "items": { "$ref": "#/definitions/Resource" } - }, - "toolConnections": { - "type": "array", - "description": "List of tool connections to external services (MCP tools, A2A, custom APIs) created during provisioning.", - "items": { "$ref": "#/definitions/ToolConnection" } - }, - "toolboxes": { - "type": "array", - "description": "List of toolboxes (Foundry Toolsets) to deploy.", - "items": { "$ref": "#/definitions/Toolbox" } - }, - "connections": { - "type": "array", - "description": "List of project connections to create via Bicep provisioning.", - "items": { "$ref": "#/definitions/Connection" } - }, - "memoryStores": { - "type": "array", - "description": "List of Foundry memory stores to provision (create-if-not-exists) during deployment. Memory stores let agents retain context across sessions via the memory_search tool.", - "items": { "$ref": "#/definitions/MemoryStore" } - }, - "startupCommand": { - "type": "string", - "description": "Command to start the agent server (e.g., 'python main.py'). Used by 'azd ai agent run' for local development." - }, - "activity": { - "$ref": "#/definitions/ActivitySettings" - }, - "kind": { - "type": "string", - "description": "The agent kind. 'hosted' for a containerized/code agent; 'prompt-voice' for a declarative (managed) speech-to-speech voice agent.", - "enum": ["hosted", "prompt-voice"] - }, +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Azure AI Agent Service Target Configuration", + "description": "Custom configuration for the Azure AI Agent Service target", + "type": "object", + "properties": { + "container": { + "$ref": "#/definitions/ContainerSettings" + }, + "deployments": { + "type": "array", + "description": "List of model deployments.", + "items": { "$ref": "#/definitions/Deployment" } + }, + "resources": { + "type": "array", + "description": "List of external resources for agent execution.", + "items": { "$ref": "#/definitions/Resource" } + }, + "toolConnections": { + "type": "array", + "description": "List of tool connections to external services (MCP tools, A2A, custom APIs) created during provisioning.", + "items": { "$ref": "#/definitions/ToolConnection" } + }, + "toolboxes": { + "type": "array", + "description": "List of toolboxes (Foundry Toolsets) to deploy.", + "items": { "$ref": "#/definitions/Toolbox" } + }, + "connections": { + "type": "array", + "description": "List of project connections to create via Bicep provisioning.", + "items": { "$ref": "#/definitions/Connection" } + }, + "memoryStores": { + "type": "array", + "description": "List of Foundry memory stores to provision (create-if-not-exists) during deployment. Memory stores let agents retain context across sessions via the memory_search tool.", + "items": { "$ref": "#/definitions/MemoryStore" } + }, + "startupCommand": { + "type": "string", + "description": "Command to start the agent server (e.g., 'python main.py'). Used by 'azd ai agent run' for local development." + }, + "activity": { + "$ref": "#/definitions/ActivitySettings" + }, + "kind": { + "type": "string", + "description": "The agent kind. 'hosted' for a containerized/code agent; 'prompt-voice' for a declarative (managed) speech-to-speech voice agent.", + "enum": ["hosted", "prompt-voice"] + }, "modelType": { "type": "string", - "description": "Voice agent (kind: prompt-voice) model-inference mode. 'managed' uses a Voice Live-hosted model; 'self_deployed' (BYOM) references an existing Foundry model deployment.", - "enum": ["managed", "self_deployed"] + "description": "Voice agent model-inference mode. 'managed' uses a Voice Live-hosted model; 'self_deployed' references a Foundry model deployment; 'hosted_agent' routes turns to a deployed hosted agent service.", + "enum": ["managed", "self_deployed", "hosted_agent"] + }, + "targetAgent": { + "$ref": "#/definitions/VoiceTargetAgent" + }, + "model": { + "type": "object", + "description": "Voice agent (kind: prompt-voice) speech-to-speech model (e.g. id: gpt-realtime).", + "properties": { + "id": { "type": "string", "minLength": 1, "pattern": "\\S", "description": "Model name for managed mode (e.g. 'gpt-realtime') or existing Foundry deployment name for self_deployed mode." } + }, + "required": ["id"], + "additionalProperties": true + }, + "instructions": { + "type": "string", + "description": "Voice agent (kind: prompt-voice) system prompt for the assistant." + }, + "voice": { + "type": "string", + "description": "Voice agent (kind: prompt-voice) output voice name (e.g. 'en-US-Ava:DragonHDLatestNeural' for an Azure Neural voice, or 'alloy' for an OpenAI realtime voice)." }, - "model": { + "structuredInputs": { "type": "object", - "description": "Voice agent (kind: prompt-voice) speech-to-speech model (e.g. id: gpt-realtime).", - "properties": { - "id": { "type": "string", "minLength": 1, "pattern": "\\S", "description": "Model name for managed mode (e.g. 'gpt-realtime') or existing Foundry deployment name for self_deployed mode." } - }, - "required": ["id"], + "description": "Voice agent (kind: prompt-voice) structured prompt inputs. Use description, defaultValue, schema, and required; azd maps defaultValue to the service wire field default_value.", "additionalProperties": true }, - "instructions": { - "type": "string", - "description": "Voice agent (kind: prompt-voice) system prompt for the assistant." + "audio": { + "$ref": "#/definitions/VoiceAudio" }, - "voice": { - "type": "string", - "description": "Voice agent (kind: prompt-voice) output voice name (e.g. 'en-US-Ava:DragonHDLatestNeural' for an Azure Neural voice, or 'alloy' for an OpenAI realtime voice)." + "outputModalities": { + "type": "array", + "description": "Voice agent (kind: prompt-voice) output modalities. Well-known values are audio, text, animation, and avatar.", + "items": { "type": "string", "minLength": 1 } }, "store": { "type": "boolean", "description": "Voice agent (kind: prompt-voice) server-side logging toggle (transcript + per-turn audio). Defaults to false when omitted." }, - "name": { - "type": "string", - "description": "The agent name." - }, - "displayName": { - "type": "string", - "description": "Optional human-friendly display name for the agent." - }, - "description": { - "type": "string", - "description": "Optional description of the agent." + "tools": { + "type": "array", + "description": "Voice agent (kind: prompt-voice) tools. Direct tool types include function, mcp, system, and toolbox.", + "items": { "type": "object", "additionalProperties": true } }, - "metadata": { + "avatar": { "type": "object", - "description": "Optional metadata key-value pairs for the agent.", + "description": "Voice agent (kind: prompt-voice) avatar configuration.", "additionalProperties": true }, - "protocols": { - "type": "array", - "description": "Invocation protocols the agent implements (e.g., responses, invocations, invocations_ws, a2a).", - "items": { "$ref": "#/definitions/ProtocolVersionRecord" } - }, - "agentEndpoint": { + "greeting": { "type": "object", - "description": "Agent endpoint configuration (protocols, version selection, auth).", + "description": "Voice agent (kind: prompt-voice) greeting configuration, such as template or llm_generated.", "additionalProperties": true }, - "agentCard": { + "handoff": { "type": "object", - "description": "A2A discovery metadata for the agent.", + "description": "Voice agent (kind: prompt-voice) handoff configuration.", "additionalProperties": true }, - "codeConfiguration": { - "$ref": "#/definitions/CodeConfiguration" + "toolChoice": { + "description": "Voice agent (kind: prompt-voice) tool choice behavior, such as none, auto, required, or a tool choice object." }, - "sessionConfiguration": { - "$ref": "#/definitions/SessionConfiguration" + "parallelToolCalls": { + "type": "boolean", + "description": "Voice agent (kind: prompt-voice) parallel tool call toggle. Currently rejected by azd for prompt voice runtime." }, - "policies": { - "type": "array", - "description": "Governance policies attached to the agent (e.g., Responsible AI).", - "items": { "$ref": "#/definitions/Policy" } + "maxOutputTokens": { + "type": ["integer", "string"], + "description": "Voice agent (kind: prompt-voice) maximum output tokens. Use an integer or a service-supported string such as inf." }, - "inputSchema": { - "type": "object", - "description": "Optional input schema for the agent.", - "additionalProperties": true - }, - "outputSchema": { - "type": "object", - "description": "Optional output schema for the agent.", - "additionalProperties": true - } - }, + "include": { + "type": "array", + "description": "Voice agent (kind: prompt-voice) extra service response fields to include.", + "items": { "type": "string" } + }, + "name": { + "type": "string", + "description": "The agent name." + }, + "displayName": { + "type": "string", + "description": "Optional human-friendly display name for the agent." + }, + "description": { + "type": "string", + "description": "Optional description of the agent." + }, + "metadata": { + "type": "object", + "description": "Optional metadata key-value pairs for the agent.", + "additionalProperties": true + }, + "protocols": { + "type": "array", + "description": "Invocation protocols the agent implements (e.g., responses, invocations, invocations_ws, a2a).", + "items": { "$ref": "#/definitions/ProtocolVersionRecord" } + }, + "agentEndpoint": { + "type": "object", + "description": "Agent endpoint configuration (protocols, version selection, auth).", + "additionalProperties": true + }, + "agentCard": { + "type": "object", + "description": "A2A discovery metadata for the agent.", + "additionalProperties": true + }, + "codeConfiguration": { + "$ref": "#/definitions/CodeConfiguration" + }, + "sessionConfiguration": { + "$ref": "#/definitions/SessionConfiguration" + }, + "policies": { + "type": "array", + "description": "Governance policies attached to the agent (e.g., Responsible AI).", + "items": { "$ref": "#/definitions/Policy" } + }, + "inputSchema": { + "type": "object", + "description": "Optional input schema for the agent.", + "additionalProperties": true + }, + "outputSchema": { + "type": "object", + "description": "Optional output schema for the agent.", + "additionalProperties": true + } + }, "additionalProperties": true, "allOf": [ { - "$comment": "A prompt-voice agent must declare a speech-to-speech model; the deploy path rejects a voice service whose model.id is missing. Keep editor/schema validation aligned with that runtime requirement.", + "$comment": "A hosted-agent voice wrapper references a deployed hosted service; other voice modes declare a model.", "if": { "properties": { - "kind": { "const": "prompt-voice" } + "kind": { "const": "prompt-voice" }, + "modelType": { "const": "hosted_agent" } }, - "required": ["kind"] + "required": ["kind", "modelType"] }, "then": { - "required": ["model"] - } - }, - { - "$comment": "The activity.publish block is shared publish metadata for Activity use cases (including simple). Digital Worker adds stricter requirements: publish must exist, publishScope must be tenant, and agenticUserTemplate is required to carry the persisted blueprint identity.", - "if": { - "properties": { - "activity": { - "properties": { - "useCase": { "const": "digital_worker" } - }, - "required": ["useCase"] - } - }, - "required": ["activity"] + "required": ["targetAgent"], + "not": { + "anyOf": [ + { "required": ["model"] }, + { "required": ["instructions"] }, + { "required": ["structuredInputs"] }, + { "required": ["tools"] }, + { "required": ["toolChoice"] }, + { "required": ["parallelToolCalls"] }, + { "required": ["maxOutputTokens"] }, + { "required": ["include"] }, + { "required": ["handoff"] } + ] + } }, - "then": { - "properties": { - "activity": { - "required": ["publish"], - "properties": { - "publish": { - "properties": { - "publishScope": { "const": "tenant" } - }, - "required": ["agenticUserTemplate"] - } - } - } + "else": { + "if": { + "properties": { "kind": { "const": "prompt-voice" } }, + "required": ["kind"] + }, + "then": { + "required": ["model"], + "not": { "required": ["targetAgent"] } } } - } + }, + { + "$comment": "The activity.publish block is shared publish metadata for Activity use cases (including simple). Digital Worker adds stricter requirements: publish must exist, publishScope must be tenant, and agenticUserTemplate is required to carry the persisted blueprint identity.", + "if": { + "properties": { + "activity": { + "properties": { + "useCase": { "const": "digital_worker" } + }, + "required": ["useCase"] + } + }, + "required": ["activity"] + }, + "then": { + "properties": { + "activity": { + "required": ["publish"], + "properties": { + "publish": { + "properties": { + "publishScope": { "const": "tenant" } + }, + "required": ["agenticUserTemplate"] + } + } + } + } + } + } ], "definitions": { - "ActivitySettings": { - "type": "object", - "description": "Activity-protocol Teams configuration. The publish block is shared metadata for Activity use cases; digital_worker applies additional constraints via allOf.", - "properties": { - "useCase": { "type": "string", "enum": ["simple", "digital_worker"] }, - "publish": { - "$ref": "#/definitions/ActivityPublishConfig", - "description": "Shared Microsoft 365 app publish metadata used by Activity use cases. For digital_worker, publishScope=tenant and agenticUserTemplate are required." - } - }, - "additionalProperties": false - }, - "ActivityPublishConfig": { + "VoiceTargetAgent": { "type": "object", + "description": "Hosted agent service used as the conversation target for a voice wrapper.", "properties": { - "publishScope": { "type": "string", "enum": ["shared", "tenant"] }, - "canRespondWithoutMention": { "type": "boolean" }, - "appVersion": { "type": "string", "minLength": 1 }, - "agentDisplayName": { "type": "string", "minLength": 1 }, - "shortDescription": { "type": "string" }, - "fullDescription": { "type": "string" }, - "developerName": { "type": "string" }, - "developerWebsiteUrl": { "type": "string", "format": "uri" }, - "privacyUrl": { "type": "string", "format": "uri" }, - "termsOfUseUrl": { "type": "string", "format": "uri" }, - "agenticUserTemplate": { "$ref": "#/definitions/AgenticUserTemplateConfig" } + "service": { "type": "string", "minLength": 1, "pattern": "\\S", "description": "azure.yaml service name of the target hosted agent." }, + "version": { "type": "string", "enum": ["deployed"], "default": "deployed", "description": "Pin the wrapper to the target version deployed by this azd environment." } }, + "required": ["service"], "additionalProperties": false }, - "AgenticUserTemplateConfig": { - "type": "object", - "properties": { - "id": { "type": "string", "minLength": 1 }, - "file": { "type": "string", "minLength": 1 }, - "schemaVersion": { "type": "string", "minLength": 1 }, - "communicationProtocol": { "type": "string", "minLength": 1 } - }, + "ActivitySettings": { + "type": "object", + "description": "Activity-protocol Teams configuration. The publish block is shared metadata for Activity use cases; digital_worker applies additional constraints via allOf.", + "properties": { + "useCase": { "type": "string", "enum": ["simple", "digital_worker"] }, + "publish": { + "$ref": "#/definitions/ActivityPublishConfig", + "description": "Shared Microsoft 365 app publish metadata used by Activity use cases. For digital_worker, publishScope=tenant and agenticUserTemplate are required." + } + }, + "additionalProperties": false + }, + "ActivityPublishConfig": { + "type": "object", + "properties": { + "publishScope": { "type": "string", "enum": ["shared", "tenant"] }, + "canRespondWithoutMention": { "type": "boolean" }, + "appVersion": { "type": "string", "minLength": 1 }, + "agentDisplayName": { "type": "string", "minLength": 1 }, + "shortDescription": { "type": "string" }, + "fullDescription": { "type": "string" }, + "developerName": { "type": "string" }, + "developerWebsiteUrl": { "type": "string", "format": "uri" }, + "privacyUrl": { "type": "string", "format": "uri" }, + "termsOfUseUrl": { "type": "string", "format": "uri" }, + "agenticUserTemplate": { "$ref": "#/definitions/AgenticUserTemplateConfig" } + }, + "additionalProperties": false + }, + "AgenticUserTemplateConfig": { + "type": "object", + "properties": { + "id": { "type": "string", "minLength": 1 }, + "file": { "type": "string", "minLength": 1 }, + "schemaVersion": { "type": "string", "minLength": 1 }, + "communicationProtocol": { "type": "string", "minLength": 1 } + }, "required": ["id", "file", "schemaVersion", "communicationProtocol"], "additionalProperties": false }, "ProtocolVersionRecord": { - "type": "object", - "description": "A protocol the agent implements, with its version.", - "properties": { - "protocol": { "type": "string", "description": "Protocol name (e.g., 'responses', 'invocations', 'invocations_ws', 'a2a')." }, - "version": { "type": "string", "description": "Protocol version." } - }, - "required": ["protocol"], - "additionalProperties": false - }, + "type": "object", + "description": "A protocol the agent implements, with its version.", + "properties": { + "protocol": { "type": "string", "description": "Protocol name (e.g., 'responses', 'invocations', 'invocations_ws', 'a2a')." }, + "version": { "type": "string", "description": "Protocol version." } + }, + "required": ["protocol"], + "additionalProperties": false + }, "CodeConfiguration": { - "type": "object", - "description": "Code deploy configuration. When present, the agent is deployed from source (ZIP) instead of a container image.", - "properties": { - "runtime": { "type": "string", "description": "Runtime identifier (e.g., 'python_3_12', 'dotnet_9')." }, - "entryPoint": { "type": "string", "description": "Entry point for the agent source." }, - "dependencyResolution": { "type": "string", "description": "Dependency resolution mode (e.g., 'bundled', 'remote_build')." } + "type": "object", + "description": "Code deploy configuration. When present, the agent is deployed from source (ZIP) instead of a container image.", + "properties": { + "runtime": { "type": "string", "description": "Runtime identifier (e.g., 'python_3_12', 'dotnet_9')." }, + "entryPoint": { "type": "string", "description": "Entry point for the agent source." }, + "dependencyResolution": { "type": "string", "description": "Dependency resolution mode (e.g., 'bundled', 'remote_build')." } }, "required": ["runtime", "entryPoint"], "additionalProperties": false }, - "SessionConfiguration": { - "type": "object", - "description": "Optional hosted-agent session runtime settings. When omitted, the service applies its defaults (idle timeout 900 seconds).", - "properties": { - "idleTimeoutSeconds": { - "type": "integer", - "description": "Idle duration in seconds before a session's sandbox is suspended. Range 300–3600 (inclusive). Defaults to 900 when omitted.", - "minimum": 300, - "maximum": 3600 - } - }, - "additionalProperties": false - }, - "Policy": { - "type": "object", - "description": "A safety or governance policy attached to the agent.", - "properties": { - "type": { "type": "string", "description": "Policy type (e.g., 'rai_policy')." }, - "raiPolicyName": { "type": "string", "description": "ARM resource ID of the RAI policy (for type 'rai_policy')." }, - "invocationsModeration": { "$ref": "#/definitions/InvocationsModeration" } - }, - "required": ["type"], - "additionalProperties": false - }, - "InvocationsModeration": { + "VoiceAudio": { "type": "object", - "description": "Configures how the content-safety proxy extracts the text it submits to the RAI policy. Only supported for agents that expose the 'invocations' protocol; without it an attached RAI policy has nothing to moderate on that path.", + "description": "Prompt voice input and output audio configuration.", "properties": { - "inputContentType": { - "type": "string", - "enum": ["json", "text"], - "description": "How the request body is encoded. Defaults to 'json'." - }, - "outputContentType": { - "type": "string", - "enum": ["json", "text"], - "description": "How the response body is encoded. Defaults to 'json'." - }, - "responseMode": { - "type": "string", - "enum": ["non_streaming", "streaming", "both"], - "description": "Response shapes the agent container can produce. This declares a capability, not an input/output switch: the proxy runs exactly one output gate per response, chosen from the actual response Content-Type." - }, - "inputPaths": { - "type": "array", - "description": "JSONPath expressions selecting request text. Required when inputContentType is 'json' or omitted.", - "minItems": 1, - "items": { "type": "string" } - }, - "outputPaths": { - "type": "array", - "description": "JSONPath expressions selecting buffered response text. Required when responseMode includes non-streaming and outputContentType is 'json' or omitted.", - "minItems": 1, - "items": { "type": "string" } - }, - "streamSelectors": { - "type": "array", - "description": "Locates text within server-sent event frames. Required when responseMode includes streaming and outputContentType is 'json' or omitted.", - "minItems": 1, - "items": { "$ref": "#/definitions/SseTextSelector" } - } + "input": { "$ref": "#/definitions/VoiceAudioInput" }, + "output": { "$ref": "#/definitions/VoiceAudioOutput" } }, - "required": ["responseMode"], - "allOf": [ - { - "if": { "$ref": "#/definitions/InvocationsInputIsJson" }, - "then": { "required": ["inputPaths"] } - }, - { - "if": { - "allOf": [ - { "properties": { "responseMode": { "enum": ["non_streaming", "both"] } }, "required": ["responseMode"] }, - { "$ref": "#/definitions/InvocationsOutputIsJson" } - ] - }, - "then": { "required": ["outputPaths"] } - }, - { - "if": { - "allOf": [ - { "properties": { "responseMode": { "enum": ["streaming", "both"] } }, "required": ["responseMode"] }, - { "$ref": "#/definitions/InvocationsOutputIsJson" } - ] - }, - "then": { "required": ["streamSelectors"] } - } - ], "additionalProperties": false }, - "InvocationsInputIsJson": { - "description": "Matches when inputContentType resolves to 'json', i.e. it is set to 'json' or omitted entirely (the default).", - "anyOf": [ - { "properties": { "inputContentType": { "const": "json" } }, "required": ["inputContentType"] }, - { "not": { "required": ["inputContentType"] } } - ] - }, - "InvocationsOutputIsJson": { - "description": "Matches when outputContentType resolves to 'json', i.e. it is set to 'json' or omitted entirely (the default).", - "anyOf": [ - { "properties": { "outputContentType": { "const": "json" } }, "required": ["outputContentType"] }, - { "not": { "required": ["outputContentType"] } } - ] - }, - "SseTextSelector": { + "VoiceAudioInput": { "type": "object", - "description": "Locates the text to moderate inside a single server-sent event frame.", "properties": { - "eventType": { "type": "string", "pattern": "\\S", "description": "SSE event name this selector applies to." }, - "textField": { "type": "string", "description": "JSONPath expression, relative to the frame payload, holding the text." } + "format": { "$ref": "#/definitions/VoiceAudioFormat" }, + "noiseReduction": { "$ref": "#/definitions/VoiceNoiseReduction" }, + "echoCancellation": { "type": "object", "additionalProperties": true }, + "turnDetection": { "$ref": "#/definitions/VoiceTurnDetection" }, + "transcription": { "$ref": "#/definitions/VoiceTranscription" } }, - "required": ["eventType"], "additionalProperties": false }, - "ContainerSettings": { + "VoiceAudioOutput": { "type": "object", - "description": "Container configuration for the Azure AI Agent Service target", "properties": { - "resources": { - "$ref": "#/definitions/ResourceSettings" - } + "format": { "$ref": "#/definitions/VoiceAudioFormat" }, + "voice": { "$ref": "#/definitions/VoiceConfig" }, + "speed": { "type": "number", "minimum": 0.25, "maximum": 1.5 } }, "additionalProperties": false }, - "ResourceSettings": { + "VoiceAudioFormat": { "type": "object", - "description": "Resource configuration for the Azure AI Agent Service target", "properties": { - "memory": { - "type": "string", - "description": "Memory allocation (e.g., '1Gi', '512Mi')", - "pattern": "^[0-9]+(\\.[0-9]+)?(Ki|Mi|Gi|Ti|Pi|Ei|k|M|G|T|P|E)?$" - }, - "cpu": { - "type": "string", - "description": "CPU allocation (e.g., '1', '500m')", - "pattern": "^[0-9]+(\\.[0-9]+)?m?$" - } + "type": { "type": "string", "enum": ["audio/pcm", "audio/pcmu", "audio/pcma"] }, + "rate": { "type": "integer", "minimum": 1 } }, + "required": ["type"], "additionalProperties": false }, - "Deployment": { - "type": "object", - "description": "A single model deployment.", - "properties": { - "name": { "type": "string", "description": "Name of the model deployment." }, - "model": { "$ref": "#/definitions/DeploymentModel" }, - "sku": { "$ref": "#/definitions/DeploymentSku" } - }, - "required": ["name", "model", "sku"], - "additionalProperties": false - }, - "DeploymentModel": { - "type": "object", - "description": "Model configuration for a model deployment.", - "properties": { - "name": { "type": "string", "description": "Model name." }, - "format": { "type": "string", "description": "Model format." }, - "version": { "type": "string", "description": "Model version." } - }, - "required": ["name", "format", "version"], - "additionalProperties": false - }, - "DeploymentSku": { + "VoiceNoiseReduction": { "type": "object", - "description": "SKU configuration for a deployment.", "properties": { - "name": { "type": "string", "description": "SKU name." }, - "capacity": { "type": "integer", "description": "SKU capacity." } + "type": { "type": "string", "description": "Well-known values include near_field, far_field, and azure_deep_noise_suppression." } }, - "required": ["name", "capacity"], + "required": ["type"], "additionalProperties": false }, - "Resource": { + "VoiceTurnDetection": { "type": "object", - "description": "External resource for agent execution.", "properties": { - "resource": { "type": "string", "description": "Resource identifier." }, - "connectionName": { "type": "string", "description": "Connection name for the resource." } + "type": { "type": "string", "description": "Well-known values include server_vad, semantic_vad, and azure_semantic_vad." }, + "threshold": { "type": "number", "exclusiveMinimum": 0, "maximum": 1 }, + "prefixPaddingMs": { "type": "integer", "minimum": 0 }, + "silenceDurationMs": { "type": "integer", "minimum": 0 }, + "createResponse": { "type": "boolean" }, + "eagerness": { "type": "string" }, + "speechDurationMs": { "type": "integer", "minimum": 0 }, + "removeFillerWords": { "type": "boolean" }, + "interruptResponse": { "type": "boolean" }, + "languages": { "type": "array", "items": { "type": "string" } }, + "autoTruncate": { "type": "boolean" } }, - "required": ["resource", "connectionName"], - "additionalProperties": false - }, - "ToolConnection": { - "type": "object", - "description": "A connection to an external service (MCP tool, A2A, custom API) created via Bicep during provisioning.", - "properties": { - "name": { "type": "string", "description": "Connection name used as project_connection_id in toolbox tools." }, - "category": { "type": "string", "description": "Connection category (e.g., 'RemoteTool')." }, - "target": { "type": "string", "description": "Target endpoint URL for the connection." }, - "authType": { - "type": "string", - "description": "Authentication type for the connection.", - "enum": ["AAD", "AccessKey", "AccountKey", "ApiKey", "CustomKeys", "ManagedIdentity", "None", "OAuth2", "PAT", "ServicePrincipal", "UsernamePassword", "ProjectManagedIdentity", "UserEntraToken", "AgenticIdentityToken"] - }, - "credentials": { - "type": "object", - "description": "Credentials for the connection. Values may contain ${ENV_VAR} references resolved at provision time." - }, - "metadata": { - "type": "object", - "description": "Additional metadata for the connection.", - "additionalProperties": { "type": "string" } - } - }, - "required": ["name", "category", "target", "authType"], + "required": ["type"], "additionalProperties": false }, - "Toolbox": { + "VoiceTranscription": { "type": "object", - "description": "A reusable collection of tools deployed as a Foundry Toolset.", "properties": { - "name": { "type": "string", "description": "Name of the toolbox." }, - "description": { "type": "string", "description": "Description of the toolbox." }, - "tools": { - "type": "array", - "description": "List of tools in the toolbox. Each tool is an object with properties passed to the Foundry Toolsets API.", - "items": { "type": "object" } - } + "model": { "type": "string" }, + "language": { "type": "string" }, + "prompt": { "type": "string" } }, - "required": ["name", "tools"], "additionalProperties": false }, - "Connection": { + "VoiceConfig": { "type": "object", - "description": "A project connection matching the Bicep ConnectionPropertiesV2 spec.", "properties": { - "name": { "type": "string", "description": "Connection name.", "pattern": "^[a-zA-Z0-9][a-zA-Z0-9_-]{2,32}$" }, - "category": { "type": "string", "description": "Connection category (e.g., 'CustomKeys', 'AzureOpenAI', 'CognitiveSearch', 'RemoteTool')." }, - "target": { "type": "string", "description": "Target endpoint URL for the connection." }, - "authType": { - "type": "string", - "description": "Authentication type.", - "enum": ["AAD", "AccessKey", "AccountKey", "AgenticIdentity", "AgenticIdentityToken", "ApiKey", "CustomKeys", "ManagedIdentity", "None", "OAuth2", "PAT", "SAS", "ServicePrincipal", "UsernamePassword", "UserEntraToken", "ProjectManagedIdentity"] - }, - "credentials": { - "type": "object", - "description": "Authentication credentials. Structure depends on authType." - }, - "metadata": { - "type": "object", - "description": "Additional metadata as key-value pairs.", - "additionalProperties": { "type": "string" } - }, - "authorizationUrl": { "type": "string", "description": "OAuth2 authorization endpoint URL (required for OAuth2 authType)." }, - "tokenUrl": { "type": "string", "description": "OAuth2 token endpoint URL (required for OAuth2 authType)." }, - "refreshUrl": { "type": "string", "description": "OAuth2 token refresh URL (optional for OAuth2 authType)." }, - "scopes": { - "type": "array", - "description": "OAuth2 scopes to request (optional for OAuth2 authType).", - "items": { "type": "string" } - }, - "audience": { "type": "string", "description": "Token audience for AAD/ProjectManagedIdentity/AgenticIdentity/AgenticIdentityToken/UserEntraToken auth types." }, - "connectorName": { "type": "string", "description": "Connector name for Oauth2 auth type." }, - "expiryTime": { "type": "string", "description": "Connection expiry time." }, - "isSharedToAll": { "type": "boolean", "description": "Whether the connection is shared to all users." }, - "sharedUserList": { - "type": "array", - "description": "List of users the connection is shared with.", - "items": { "type": "string" } - }, - "peRequirement": { "type": "string", "description": "Private endpoint requirement." }, - "peStatus": { "type": "string", "description": "Private endpoint status." }, - "useWorkspaceManagedIdentity": { "type": "boolean", "description": "Whether to use workspace managed identity." }, - "error": { "type": "string", "description": "Error information." } + "type": { "type": "string", "description": "Voice provider type, such as openai, azure_standard, or azure-standard." }, + "name": { "type": "string" }, + "style": { "type": "string" }, + "pitch": { "type": "string" }, + "rate": { "type": "string" }, + "locale": { "type": "string" }, + "volume": { "type": "string" } }, - "required": ["name", "category", "target", "authType"], + "required": ["type", "name"], "additionalProperties": false }, - "MemoryStore": { - "type": "object", - "description": "A Foundry memory store provisioned (create-if-not-exists) during deployment. Backs the agent's memory_search tool so the agent can retain context across sessions.", - "properties": { - "name": { "type": "string", "description": "Name of the memory store." }, - "description": { "type": "string", "description": "Description of the memory store." }, - "chatModel": { "type": "string", "description": "Chat model deployment name used by the memory store (must exist in the Foundry project)." }, - "embeddingModel": { "type": "string", "description": "Embedding model deployment name used by the memory store (must exist in the Foundry project)." }, - "options": { - "type": "object", - "description": "Optional extraction and retention settings for the memory store.", - "properties": { - "chatSummaryEnabled": { "type": "boolean", "description": "Enable rolling chat-summary memory." }, - "userProfileEnabled": { "type": "boolean", "description": "Enable durable user-profile memory." }, - "proceduralMemoryEnabled": { "type": "boolean", "description": "Enable procedural (how-to) memory." }, - "defaultTtlSeconds": { "type": "integer", "description": "Default time-to-live (seconds) for new memory entries. 0 means no expiration." }, - "userProfileDetails": { "type": "string", "description": "Guidance on what user-profile information the agent should retain or avoid." } - }, - "additionalProperties": false - } - }, - "required": ["name", "chatModel", "embeddingModel"], - "additionalProperties": false - } - } -} + "SessionConfiguration": { + "type": "object", + "description": "Optional hosted-agent session runtime settings. When omitted, the service applies its defaults (idle timeout 900 seconds).", + "properties": { + "idleTimeoutSeconds": { + "type": "integer", + "description": "Idle duration in seconds before a session's sandbox is suspended. Range 300–3600 (inclusive). Defaults to 900 when omitted.", + "minimum": 300, + "maximum": 3600 + } + }, + "additionalProperties": false + }, + "Policy": { + "type": "object", + "description": "A safety or governance policy attached to the agent.", + "properties": { + "type": { "type": "string", "description": "Policy type (e.g., 'rai_policy')." }, + "raiPolicyName": { "type": "string", "description": "ARM resource ID of the RAI policy (for type 'rai_policy')." }, + "invocationsModeration": { "$ref": "#/definitions/InvocationsModeration" } + }, + "required": ["type"], + "additionalProperties": false + }, + "InvocationsModeration": { + "type": "object", + "description": "Configures how the content-safety proxy extracts the text it submits to the RAI policy. Only supported for agents that expose the 'invocations' protocol; without it an attached RAI policy has nothing to moderate on that path.", + "properties": { + "inputContentType": { + "type": "string", + "enum": ["json", "text"], + "description": "How the request body is encoded. Defaults to 'json'." + }, + "outputContentType": { + "type": "string", + "enum": ["json", "text"], + "description": "How the response body is encoded. Defaults to 'json'." + }, + "responseMode": { + "type": "string", + "enum": ["non_streaming", "streaming", "both"], + "description": "Response shapes the agent container can produce. This declares a capability, not an input/output switch: the proxy runs exactly one output gate per response, chosen from the actual response Content-Type." + }, + "inputPaths": { + "type": "array", + "description": "JSONPath expressions selecting request text. Required when inputContentType is 'json' or omitted.", + "minItems": 1, + "items": { "type": "string" } + }, + "outputPaths": { + "type": "array", + "description": "JSONPath expressions selecting buffered response text. Required when responseMode includes non-streaming and outputContentType is 'json' or omitted.", + "minItems": 1, + "items": { "type": "string" } + }, + "streamSelectors": { + "type": "array", + "description": "Locates text within server-sent event frames. Required when responseMode includes streaming and outputContentType is 'json' or omitted.", + "minItems": 1, + "items": { "$ref": "#/definitions/SseTextSelector" } + } + }, + "required": ["responseMode"], + "allOf": [ + { + "if": { "$ref": "#/definitions/InvocationsInputIsJson" }, + "then": { "required": ["inputPaths"] } + }, + { + "if": { + "allOf": [ + { "properties": { "responseMode": { "enum": ["non_streaming", "both"] } }, "required": ["responseMode"] }, + { "$ref": "#/definitions/InvocationsOutputIsJson" } + ] + }, + "then": { "required": ["outputPaths"] } + }, + { + "if": { + "allOf": [ + { "properties": { "responseMode": { "enum": ["streaming", "both"] } }, "required": ["responseMode"] }, + { "$ref": "#/definitions/InvocationsOutputIsJson" } + ] + }, + "then": { "required": ["streamSelectors"] } + } + ], + "additionalProperties": false + }, + "InvocationsInputIsJson": { + "description": "Matches when inputContentType resolves to 'json', i.e. it is set to 'json' or omitted entirely (the default).", + "anyOf": [ + { "properties": { "inputContentType": { "const": "json" } }, "required": ["inputContentType"] }, + { "not": { "required": ["inputContentType"] } } + ] + }, + "InvocationsOutputIsJson": { + "description": "Matches when outputContentType resolves to 'json', i.e. it is set to 'json' or omitted entirely (the default).", + "anyOf": [ + { "properties": { "outputContentType": { "const": "json" } }, "required": ["outputContentType"] }, + { "not": { "required": ["outputContentType"] } } + ] + }, + "SseTextSelector": { + "type": "object", + "description": "Locates the text to moderate inside a single server-sent event frame.", + "properties": { + "eventType": { "type": "string", "pattern": "\\S", "description": "SSE event name this selector applies to." }, + "textField": { "type": "string", "description": "JSONPath expression, relative to the frame payload, holding the text." } + }, + "required": ["eventType"], + "additionalProperties": false + }, + "ContainerSettings": { + "type": "object", + "description": "Container configuration for the Azure AI Agent Service target", + "properties": { + "resources": { + "$ref": "#/definitions/ResourceSettings" + } + }, + "additionalProperties": false + }, + "ResourceSettings": { + "type": "object", + "description": "Resource configuration for the Azure AI Agent Service target", + "properties": { + "memory": { + "type": "string", + "description": "Memory allocation (e.g., '1Gi', '512Mi')", + "pattern": "^[0-9]+(\\.[0-9]+)?(Ki|Mi|Gi|Ti|Pi|Ei|k|M|G|T|P|E)?$" + }, + "cpu": { + "type": "string", + "description": "CPU allocation (e.g., '1', '500m')", + "pattern": "^[0-9]+(\\.[0-9]+)?m?$" + } + }, + "additionalProperties": false + }, + "Deployment": { + "type": "object", + "description": "A single model deployment.", + "properties": { + "name": { "type": "string", "description": "Name of the model deployment." }, + "model": { "$ref": "#/definitions/DeploymentModel" }, + "sku": { "$ref": "#/definitions/DeploymentSku" } + }, + "required": ["name", "model", "sku"], + "additionalProperties": false + }, + "DeploymentModel": { + "type": "object", + "description": "Model configuration for a model deployment.", + "properties": { + "name": { "type": "string", "description": "Model name." }, + "format": { "type": "string", "description": "Model format." }, + "version": { "type": "string", "description": "Model version." } + }, + "required": ["name", "format", "version"], + "additionalProperties": false + }, + "DeploymentSku": { + "type": "object", + "description": "SKU configuration for a deployment.", + "properties": { + "name": { "type": "string", "description": "SKU name." }, + "capacity": { "type": "integer", "description": "SKU capacity." } + }, + "required": ["name", "capacity"], + "additionalProperties": false + }, + "Resource": { + "type": "object", + "description": "External resource for agent execution.", + "properties": { + "resource": { "type": "string", "description": "Resource identifier." }, + "connectionName": { "type": "string", "description": "Connection name for the resource." } + }, + "required": ["resource", "connectionName"], + "additionalProperties": false + }, + "ToolConnection": { + "type": "object", + "description": "A connection to an external service (MCP tool, A2A, custom API) created via Bicep during provisioning.", + "properties": { + "name": { "type": "string", "description": "Connection name used as project_connection_id in toolbox tools." }, + "category": { "type": "string", "description": "Connection category (e.g., 'RemoteTool')." }, + "target": { "type": "string", "description": "Target endpoint URL for the connection." }, + "authType": { + "type": "string", + "description": "Authentication type for the connection.", + "enum": ["AAD", "AccessKey", "AccountKey", "ApiKey", "CustomKeys", "ManagedIdentity", "None", "OAuth2", "PAT", "ServicePrincipal", "UsernamePassword", "ProjectManagedIdentity", "UserEntraToken", "AgenticIdentityToken"] + }, + "credentials": { + "type": "object", + "description": "Credentials for the connection. Values may contain ${ENV_VAR} references resolved at provision time." + }, + "metadata": { + "type": "object", + "description": "Additional metadata for the connection.", + "additionalProperties": { "type": "string" } + } + }, + "required": ["name", "category", "target", "authType"], + "additionalProperties": false + }, + "Toolbox": { + "type": "object", + "description": "A reusable collection of tools deployed as a Foundry Toolset.", + "properties": { + "name": { "type": "string", "description": "Name of the toolbox." }, + "description": { "type": "string", "description": "Description of the toolbox." }, + "tools": { + "type": "array", + "description": "List of tools in the toolbox. Each tool is an object with properties passed to the Foundry Toolsets API.", + "items": { "type": "object" } + } + }, + "required": ["name", "tools"], + "additionalProperties": false + }, + "Connection": { + "type": "object", + "description": "A project connection matching the Bicep ConnectionPropertiesV2 spec.", + "properties": { + "name": { "type": "string", "description": "Connection name.", "pattern": "^[a-zA-Z0-9][a-zA-Z0-9_-]{2,32}$" }, + "category": { "type": "string", "description": "Connection category (e.g., 'CustomKeys', 'AzureOpenAI', 'CognitiveSearch', 'RemoteTool')." }, + "target": { "type": "string", "description": "Target endpoint URL for the connection." }, + "authType": { + "type": "string", + "description": "Authentication type.", + "enum": ["AAD", "AccessKey", "AccountKey", "AgenticIdentity", "AgenticIdentityToken", "ApiKey", "CustomKeys", "ManagedIdentity", "None", "OAuth2", "PAT", "SAS", "ServicePrincipal", "UsernamePassword", "UserEntraToken", "ProjectManagedIdentity"] + }, + "credentials": { + "type": "object", + "description": "Authentication credentials. Structure depends on authType." + }, + "metadata": { + "type": "object", + "description": "Additional metadata as key-value pairs.", + "additionalProperties": { "type": "string" } + }, + "authorizationUrl": { "type": "string", "description": "OAuth2 authorization endpoint URL (required for OAuth2 authType)." }, + "tokenUrl": { "type": "string", "description": "OAuth2 token endpoint URL (required for OAuth2 authType)." }, + "refreshUrl": { "type": "string", "description": "OAuth2 token refresh URL (optional for OAuth2 authType)." }, + "scopes": { + "type": "array", + "description": "OAuth2 scopes to request (optional for OAuth2 authType).", + "items": { "type": "string" } + }, + "audience": { "type": "string", "description": "Token audience for AAD/ProjectManagedIdentity/AgenticIdentity/AgenticIdentityToken/UserEntraToken auth types." }, + "connectorName": { "type": "string", "description": "Connector name for Oauth2 auth type." }, + "expiryTime": { "type": "string", "description": "Connection expiry time." }, + "isSharedToAll": { "type": "boolean", "description": "Whether the connection is shared to all users." }, + "sharedUserList": { + "type": "array", + "description": "List of users the connection is shared with.", + "items": { "type": "string" } + }, + "peRequirement": { "type": "string", "description": "Private endpoint requirement." }, + "peStatus": { "type": "string", "description": "Private endpoint status." }, + "useWorkspaceManagedIdentity": { "type": "boolean", "description": "Whether to use workspace managed identity." }, + "error": { "type": "string", "description": "Error information." } + }, + "required": ["name", "category", "target", "authType"], + "additionalProperties": false + }, + "MemoryStore": { + "type": "object", + "description": "A Foundry memory store provisioned (create-if-not-exists) during deployment. Backs the agent's memory_search tool so the agent can retain context across sessions.", + "properties": { + "name": { "type": "string", "description": "Name of the memory store." }, + "description": { "type": "string", "description": "Description of the memory store." }, + "chatModel": { "type": "string", "description": "Chat model deployment name used by the memory store (must exist in the Foundry project)." }, + "embeddingModel": { "type": "string", "description": "Embedding model deployment name used by the memory store (must exist in the Foundry project)." }, + "options": { + "type": "object", + "description": "Optional extraction and retention settings for the memory store.", + "properties": { + "chatSummaryEnabled": { "type": "boolean", "description": "Enable rolling chat-summary memory." }, + "userProfileEnabled": { "type": "boolean", "description": "Enable durable user-profile memory." }, + "proceduralMemoryEnabled": { "type": "boolean", "description": "Enable procedural (how-to) memory." }, + "defaultTtlSeconds": { "type": "integer", "description": "Default time-to-live (seconds) for new memory entries. 0 means no expiration." }, + "userProfileDetails": { "type": "string", "description": "Guidance on what user-profile information the agent should retain or avoid." } + }, + "additionalProperties": false + } + }, + "required": ["name", "chatModel", "embeddingModel"], + "additionalProperties": false + } + } +}