From a77f9ba58e7b3d23303cb8955111260920f13736 Mon Sep 17 00:00:00 2001 From: Damon Barry Date: Thu, 28 May 2026 15:51:10 -0700 Subject: [PATCH 1/6] Fix docker host mismatch (#7506) When the ISA95 smoke test pipeline generates the config file for aziot-edged, it generates slightly different values for `agent.config.auth.serveraddress` and `agent.config.image`. In the latter, it appends `:443` to the container registry's address. It seems that some recent change to the test environment has caused Docker to complain that serveraddress and image are different and then fail the pull operation. This leaves IoT Edge in an inconsistent state and the tests fail. This change removes the logic that appends the port in the image. It also removes trailing whitespace in the script. I confirmed that the ISA95 smoke tests pass with this change. ## Azure IoT Edge PR checklist: --- scripts/linux/nested-edge-deploy-agent.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/linux/nested-edge-deploy-agent.sh b/scripts/linux/nested-edge-deploy-agent.sh index 1a6841497eb..c2f248907c8 100644 --- a/scripts/linux/nested-edge-deploy-agent.sh +++ b/scripts/linux/nested-edge-deploy-agent.sh @@ -45,11 +45,11 @@ function setup_iotedge() { if [ ! -z $PARENT_NAME ]; then echo "image = \"\$upstream:443/microsoft/azureiotedge-agent:$ARTIFACT_IMAGE_BUILD_NUMBER-linux-$image_architecture_label\"" | sudo tee -a /etc/aziot/config.toml else - echo "image = \"${CONTAINER_REGISTRY}:443/microsoft/azureiotedge-agent:$ARTIFACT_IMAGE_BUILD_NUMBER-linux-$image_architecture_label\"" | sudo tee -a /etc/aziot/config.toml - fi + echo "image = \"${CONTAINER_REGISTRY}/microsoft/azureiotedge-agent:$ARTIFACT_IMAGE_BUILD_NUMBER-linux-$image_architecture_label\"" | sudo tee -a /etc/aziot/config.toml + fi echo "createOptions = { }" | sudo tee -a /etc/aziot/config.toml echo "" | sudo tee -a /etc/aziot/config.toml - + if [ -z $PARENT_NAME ]; then echo "[agent.config.auth]" | sudo tee -a /etc/aziot/config.toml echo "serveraddress = \"${CONTAINER_REGISTRY}\"" | sudo tee -a /etc/aziot/config.toml @@ -61,7 +61,7 @@ function setup_iotedge() { if [ ! -z $PROXY_ADDRESS ]; then echo "Configuring the bootstrapping edgeAgent to use http proxy" echo "[agent.env]" | sudo tee -a /etc/aziot/config.toml - echo "https_proxy = \"${PROXY_ADDRESS}\"" | sudo tee -a /etc/aziot/config.toml + echo "https_proxy = \"${PROXY_ADDRESS}\"" | sudo tee -a /etc/aziot/config.toml echo "" | sudo tee -a /etc/aziot/config.toml echo "Adding proxy configuration to docker" @@ -77,7 +77,7 @@ function setup_iotedge() { { echo "[Service]"; echo "Environment=HTTPS_PROXY=${PROXY_ADDRESS}"; } | sudo tee /etc/systemd/system/aziot-identityd.service.d/proxy.conf - sudo systemctl daemon-reload + sudo systemctl daemon-reload echo "Adding proxy configuration to IoT Edge daemon" sudo mkdir -p /etc/systemd/system/aziot-edged.service.d/ @@ -173,7 +173,7 @@ function process_args() { elif [ $saveNextArg -eq 16 ]; then CONNECTION_STRING="$arg" saveNextArg=0 - # 5/22/2024 - Temporary work around the issue where the az cli command cannot authorize itself within *.sh script using the service principal's service connection + # 5/22/2024 - Temporary work around the issue where the az cli command cannot authorize itself within *.sh script using the service principal's service connection # elif [ $saveNextArg -eq 17 ]; then # DEVICE_ID="$arg" # saveNextArg=0 From 260c5e862665e1519c04dda2dc2e6ca88e390521 Mon Sep 17 00:00:00 2001 From: Damon Barry Date: Mon, 20 Jul 2026 10:45:26 -0700 Subject: [PATCH 2/6] Fix typo in test variable --- builds/checkin/e2e-checkin.yaml | 2 +- builds/e2e/e2e.yaml | 28 +++++++++---------- builds/e2e/isa-95-smoke-test.yaml | 2 +- builds/e2e/nested-e2e.yaml | 2 +- builds/e2e/templates/e2e-setup.yaml | 4 +-- builds/service/service-deployment.yaml | 2 +- .../Context.cs | 4 +-- 7 files changed, 22 insertions(+), 22 deletions(-) diff --git a/builds/checkin/e2e-checkin.yaml b/builds/checkin/e2e-checkin.yaml index e0d541a20ee..61eb5581bf8 100644 --- a/builds/checkin/e2e-checkin.yaml +++ b/builds/checkin/e2e-checkin.yaml @@ -75,7 +75,7 @@ stages: containerRegistryUsername: '$(acrUsername)' dpsIdScope: '$(dps.idScope)' eventHubName: '$(eventhub.name)' - eventHubFullyQualitifiedNamespace: '$(eventhub.namespace.fqdn)' + eventHubFullyQualifiedNamespace: '$(eventhub.namespace.fqdn)' iotHubHostName: '$(iothub.hostname)' iotHubResourceId: '$(iothub.resourceid)' rootCaCertificate: '$(rootCaCertificate)' diff --git a/builds/e2e/e2e.yaml b/builds/e2e/e2e.yaml index 1affa74a6c9..43ab7a10a46 100644 --- a/builds/e2e/e2e.yaml +++ b/builds/e2e/e2e.yaml @@ -80,7 +80,7 @@ jobs: containerRegistryUsername: '$(containerRegistryUsername)' dpsIdScope: '$(dps.idScope)' eventHubName: '$(eventhub.name)' - eventHubFullyQualitifiedNamespace: '$(eventhub.namespace.fqdn)' + eventHubFullyQualifiedNamespace: '$(eventhub.namespace.fqdn)' iotHubHostName: '$(iothub.hostname)' iotHubResourceId: '$(iothub.resourceid)' rootCaCertificate: '$(rootCaCertificate)' @@ -131,7 +131,7 @@ jobs: containerRegistryUsername: '$(containerRegistryUsername)' dpsIdScope: '$(dps.idScope)' eventHubName: '$(eventhub.name)' - eventHubFullyQualitifiedNamespace: '$(eventhub.namespace.fqdn)' + eventHubFullyQualifiedNamespace: '$(eventhub.namespace.fqdn)' iotHubHostName: '$(iothub.hostname)' iotHubResourceId: '$(iothub.resourceid)' rootCaCertificate: '$(rootCaCertificate)' @@ -179,7 +179,7 @@ jobs: containerRegistryUsername: '$(containerRegistryUsername)' dpsIdScope: '$(dps.idScope)' eventHubName: '$(eventhub.name)' - eventHubFullyQualitifiedNamespace: '$(eventhub.namespace.fqdn)' + eventHubFullyQualifiedNamespace: '$(eventhub.namespace.fqdn)' iotHubHostName: '$(iothub.hostname)' iotHubResourceId: '$(iothub.resourceid)' rootCaCertificate: '$(rootCaCertificate)' @@ -229,7 +229,7 @@ jobs: containerRegistryUsername: '$(containerRegistryUsername)' dpsIdScope: '$(dps.idScope)' eventHubName: '$(eventhub.name)' - eventHubFullyQualitifiedNamespace: '$(eventhub.namespace.fqdn)' + eventHubFullyQualifiedNamespace: '$(eventhub.namespace.fqdn)' iotHubHostName: '$(iothub.hostname)' iotHubResourceId: '$(iothub.resourceid)' rootCaCertificate: '$(rootCaCertificate)' @@ -280,7 +280,7 @@ jobs: containerRegistryUsername: '$(containerRegistryUsername)' dpsIdScope: '$(dps.idScope)' eventHubName: '$(eventhub.name)' - eventHubFullyQualitifiedNamespace: '$(eventhub.namespace.fqdn)' + eventHubFullyQualifiedNamespace: '$(eventhub.namespace.fqdn)' iotHubHostName: '$(iothub.hostname)' iotHubResourceId: '$(iothub.resourceid)' rootCaCertificate: '$(rootCaCertificate)' @@ -330,7 +330,7 @@ jobs: containerRegistryUsername: '$(containerRegistryUsername)' dpsIdScope: '$(dps.idScope)' eventHubName: '$(eventhub.name)' - eventHubFullyQualitifiedNamespace: '$(eventhub.namespace.fqdn)' + eventHubFullyQualifiedNamespace: '$(eventhub.namespace.fqdn)' iotHubHostName: '$(iothub.hostname)' iotHubResourceId: '$(iothub.resourceid)' rootCaCertificate: '$(rootCaCertificate)' @@ -380,7 +380,7 @@ jobs: containerRegistryUsername: '$(containerRegistryUsername)' dpsIdScope: '$(dps.idScope)' eventHubName: '$(eventhub.name)' - eventHubFullyQualitifiedNamespace: '$(eventhub.namespace.fqdn)' + eventHubFullyQualifiedNamespace: '$(eventhub.namespace.fqdn)' iotHubHostName: '$(iothub.hostname)' iotHubResourceId: '$(iothub.resourceid)' rootCaCertificate: '$(rootCaCertificate)' @@ -428,7 +428,7 @@ jobs: containerRegistryUsername: '$(containerRegistryUsername)' dpsIdScope: '$(dps.idScope)' eventHubName: '$(eventhub.name)' - eventHubFullyQualitifiedNamespace: '$(eventhub.namespace.fqdn)' + eventHubFullyQualifiedNamespace: '$(eventhub.namespace.fqdn)' iotHubHostName: '$(iothub.hostname)' iotHubResourceId: '$(iothub.resourceid)' rootCaCertificate: '$(rootCaCertificate)' @@ -491,7 +491,7 @@ jobs: containerRegistryUsername: '$(containerRegistryUsername)' dpsIdScope: '$(dps.idScope)' eventHubName: '$(eventhub.name)' - eventHubFullyQualitifiedNamespace: '$(eventhub.namespace.fqdn)' + eventHubFullyQualifiedNamespace: '$(eventhub.namespace.fqdn)' iotHubHostName: '$(iothub.hostname)' iotHubResourceId: '$(iothub.resourceid)' rootCaCertificate: '$(rootCaCertificate)' @@ -539,7 +539,7 @@ jobs: containerRegistryUsername: '$(containerRegistryUsername)' dpsIdScope: '$(dps.idScope)' eventHubName: '$(eventhub.name)' - eventHubFullyQualitifiedNamespace: '$(eventhub.namespace.fqdn)' + eventHubFullyQualifiedNamespace: '$(eventhub.namespace.fqdn)' iotHubHostName: '$(iothub.hostname)' iotHubResourceId: '$(iothub.resourceid)' rootCaCertificate: '$(rootCaCertificate)' @@ -587,7 +587,7 @@ jobs: containerRegistryUsername: '$(containerRegistryUsername)' dpsIdScope: '$(dps.idScope)' eventHubName: '$(eventhub.name)' - eventHubFullyQualitifiedNamespace: '$(eventhub.namespace.fqdn)' + eventHubFullyQualifiedNamespace: '$(eventhub.namespace.fqdn)' iotHubHostName: '$(iothub.hostname)' iotHubResourceId: '$(iothub.resourceid)' rootCaCertificate: '$(rootCaCertificate)' @@ -637,7 +637,7 @@ jobs: containerRegistryUsername: '$(containerRegistryUsername)' dpsIdScope: '$(dps.idScope)' eventHubName: '$(eventhub.name)' - eventHubFullyQualitifiedNamespace: '$(eventhub.namespace.fqdn)' + eventHubFullyQualifiedNamespace: '$(eventhub.namespace.fqdn)' iotHubHostName: '$(iothub.hostname)' iotHubResourceId: '$(iothub.resourceid)' rootCaCertificate: '$(rootCaCertificate)' @@ -692,7 +692,7 @@ jobs: containerRegistryUsername: '$(containerRegistryUsername)' dpsIdScope: '$(dps.idScope)' eventHubName: '$(eventhub.name)' - eventHubFullyQualitifiedNamespace: '$(eventhub.namespace.fqdn)' + eventHubFullyQualifiedNamespace: '$(eventhub.namespace.fqdn)' iotHubHostName: '$(iothub.hostname)' iotHubResourceId: '$(iothub.resourceid)' rootCaCertificate: '$(rootCaCertificate)' @@ -745,7 +745,7 @@ jobs: containerRegistryUsername: '$(containerRegistryUsername)' dpsIdScope: '$(dps.idScope)' eventHubName: '$(eventhub.name)' - eventHubFullyQualitifiedNamespace: '$(eventhub.namespace.fqdn)' + eventHubFullyQualifiedNamespace: '$(eventhub.namespace.fqdn)' iotHubHostName: '$(iothub.hostname)' iotHubResourceId: '$(iothub.resourceid)' rootCaCertificate: '$(rootCaCertificate)' diff --git a/builds/e2e/isa-95-smoke-test.yaml b/builds/e2e/isa-95-smoke-test.yaml index ec5dd32fd14..6eeee1fc4ef 100644 --- a/builds/e2e/isa-95-smoke-test.yaml +++ b/builds/e2e/isa-95-smoke-test.yaml @@ -327,7 +327,7 @@ stages: containerRegistryServer: '$(containerRegistryServer)' containerRegistryUsername: '$(containerRegistryUsername)' eventHubName: '$(eventhub.name)' - eventHubFullyQualitifiedNamespace: '$(eventhub.namespace.fqdn)' + eventHubFullyQualifiedNamespace: '$(eventhub.namespace.fqdn)' iotHubHostName: '$(iothub.hostname)' rootCaCertificate: '$(rootCaCertificate)' rootCaKey: '$(rootCaKey)' diff --git a/builds/e2e/nested-e2e.yaml b/builds/e2e/nested-e2e.yaml index 0ac4ebe9309..eb82081e697 100644 --- a/builds/e2e/nested-e2e.yaml +++ b/builds/e2e/nested-e2e.yaml @@ -103,7 +103,7 @@ stages: containerRegistryServer: '$(containerRegistryServer)' containerRegistryUsername: '$(containerRegistryUsername)' eventHubName: '$(eventhub.name)' - eventHubFullyQualitifiedNamespace: '$(eventhub.namespace.fqdn)' + eventHubFullyQualifiedNamespace: '$(eventhub.namespace.fqdn)' iotHubHostName: '$(iothub.hostname)' iotHubResourceId: '$(iothub.resourceid)' rootCaCertificate: '$(rootCaCertificate)' diff --git a/builds/e2e/templates/e2e-setup.yaml b/builds/e2e/templates/e2e-setup.yaml index 53c3e4933bf..ac9a730a51d 100644 --- a/builds/e2e/templates/e2e-setup.yaml +++ b/builds/e2e/templates/e2e-setup.yaml @@ -3,7 +3,7 @@ parameters: containerRegistryUsername: '' dpsIdScope: 'optional' eventHubName: '' - eventHubFullyQualitifiedNamespace: '' + eventHubFullyQualifiedNamespace: '' iotHubHostName: '' iotHubResourceId: 'optional' rootCaCertificate: '' @@ -168,7 +168,7 @@ steps: $context = @{ eventHubName = '${{ parameters.eventHubName }}'; - eventHubFullyQualitifiedNamespace = '${{ parameters.eventHubFullyQualitifiedNamespace }}'; + eventHubFullyQualifiedNamespace = '${{ parameters.eventHubFullyQualifiedNamespace }}'; iotHubHostName = '${{ parameters.iotHubHostName }}'; nestededge = "$nestededge"; dpsIdScope = '${{ parameters.dpsIdScope }}'; diff --git a/builds/service/service-deployment.yaml b/builds/service/service-deployment.yaml index 754056177f0..ff4b434bbf1 100644 --- a/builds/service/service-deployment.yaml +++ b/builds/service/service-deployment.yaml @@ -56,7 +56,7 @@ steps: $context = @{ logFile = Join-Path '$(binDir)' 'testoutput.log'; eventHubName = '$(eventhub.name)'; - eventHubFullyQualitifiedNamespace = '$(eventhub.namespace.hostname)'; + eventHubFullyQualifiedNamespace = '$(eventhub.namespace.hostname)'; iotHubHostName = '$(iothub.hostname)'; caCertScriptPath = Convert-Path '$(Build.SourcesDirectory)/tools/CACertificates'; rootCaCertificatePath = Convert-Path '$(certsDir)/rsa_root_ca.cert.pem'; diff --git a/test/Microsoft.Azure.Devices.Edge.Test.Common/Context.cs b/test/Microsoft.Azure.Devices.Edge.Test.Common/Context.cs index 5e754982caa..5ec5a1a6795 100644 --- a/test/Microsoft.Azure.Devices.Edge.Test.Common/Context.cs +++ b/test/Microsoft.Azure.Devices.Edge.Test.Common/Context.cs @@ -91,8 +91,8 @@ IEnumerable GetAndValidateRegistries() this.DiagnosticsImage = Option.Maybe(Get("diagnosticsImage")); this.EventHubName = Get("eventHubName"); Preconditions.CheckArgument(!string.IsNullOrWhiteSpace(this.EventHubName), $"eventHubName is missing from environment or context.json."); - this.EventHubNamespace = Get("eventHubFullyQualitifiedNamespace"); - Preconditions.CheckArgument(!string.IsNullOrWhiteSpace(this.EventHubNamespace), $"eventHubFullyQualitifiedNamespace is missing from environment or context.json."); + this.EventHubNamespace = Get("eventHubFullyQualifiedNamespace"); + Preconditions.CheckArgument(!string.IsNullOrWhiteSpace(this.EventHubNamespace), $"eventHubFullyQualifiedNamespace is missing from environment or context.json."); this.LogFile = Option.Maybe(Get("logFile")); this.MethodReceiverImage = Option.Maybe(Get("methodReceiverImage")); this.MethodSenderImage = Option.Maybe(Get("methodSenderImage")); From b7a0a68db7015f86671a45f419ee4b4cbd077a4e Mon Sep 17 00:00:00 2001 From: Damon Barry Date: Wed, 24 Jun 2026 12:01:55 -0700 Subject: [PATCH 3/6] Use Ubuntu 24.04 artifacts in ARM64 ISA95 smoke tests (#7519) The agents that run the ISA95 smoke tests for ARM64 are running Ubuntu 24.04, but the tests are failing because they try to install IoT Edge binaries for Ubuntu 22.04. This change corrects the problem. To test, I ran the ISA95 smoke test pipeline and confirmed that it passes. ## Azure IoT Edge PR checklist: --- builds/e2e/isa-95-smoke-test.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/builds/e2e/isa-95-smoke-test.yaml b/builds/e2e/isa-95-smoke-test.yaml index 6eeee1fc4ef..eecdd4d6191 100644 --- a/builds/e2e/isa-95-smoke-test.yaml +++ b/builds/e2e/isa-95-smoke-test.yaml @@ -77,8 +77,8 @@ stages: artifactName: iotedged-ubuntu24.04-amd64 arm64: arch: ARM64 - identityArtifactName: aziot-identity-ubuntu22.04-aarch64 - artifactName: iotedged-ubuntu22.04-aarch64 + identityArtifactName: aziot-identity-ubuntu24.04-aarch64 + artifactName: iotedged-ubuntu24.04-aarch64 variables: level: 5 upstreamProtocol: amqp @@ -145,8 +145,8 @@ stages: parentDeviceId: $(x64ParentDeviceId) arm64: arch: ARM64 - identityArtifactName: aziot-identity-ubuntu22.04-aarch64 - artifactName: iotedged-ubuntu22.04-aarch64 + identityArtifactName: aziot-identity-ubuntu24.04-aarch64 + artifactName: iotedged-ubuntu24.04-aarch64 parentName: $(arm64ParentName) parentDeviceId: $(arm64ParentDeviceId) pool: @@ -206,8 +206,8 @@ stages: parentDeviceId: $(x64ParentDeviceId) arm64: arch: ARM64 - identityArtifactName: aziot-identity-ubuntu22.04-aarch64 - artifactName: iotedged-ubuntu22.04-aarch64 + identityArtifactName: aziot-identity-ubuntu24.04-aarch64 + artifactName: iotedged-ubuntu24.04-aarch64 parentName: $(arm64ParentName) parentDeviceId: $(arm64ParentDeviceId) pool: From 60b9e7d10e3fc7b5e867a6f61e2a2f2185d77765 Mon Sep 17 00:00:00 2001 From: John Lian Date: Tue, 14 Jul 2026 12:13:44 -0700 Subject: [PATCH 4/6] e2e: fix RouteMessageL3LeafToL4Module by waiting for nested deployment reconcile (#7523) ## Summary Fixes the consistently failing `RouteMessageL3LeafToL4Module` nested end-to-end test. The failure is a test race: the AMQP test group starts before the L4/L5 deployment has finished reconciling, so the L4 edgeHub restart triggered by that deployment lands in the middle of the test. The core change is a reconcile-wait guard after `set-modules`. It also keeps two supporting changes that made the failure debuggable and the repro reliable. ## Root cause The `RouteMessageL3LeafToL4Module` timeout is caused by a missing synchronization step in the test pipeline, not by a network fault. After the MQTT tests, the pipeline calls `az iot edge set-modules` for L5 and L4 and then starts the AMQP tests immediately. `set-modules` only updates the desired twin; it does not wait for edgeAgent to apply the deployment. The L4 deployment changes the upstream protocol, edgeHub configuration/routes, and modules, so L4 edgeHub restarts after the AMQP tests have already started. L3 loses its parent connection for about 15 seconds during that restart, and the in-flight device-to-module message is delayed long enough to miss the test's 300-second receive window. Verified in run [171488751](https://msazure.visualstudio.com/One/_build/results?buildId=171488751&view=results): L3 lost its parent at `03:11:37.193` and reconnected at `03:11:52.212`, a 15.018-second outage coinciding with the L4 edgeHub restart. The `iotedgeApiProxy` / `IoTEdgeAPIProxy` casing difference between the MQTT and AMQP deployment files adds reconciliation churn, but it is not the complete cause; any deployment change that restarts edgeHub would expose the same missing wait. Normalizing the casing is not the fix. `RouteMessageL3LeafToL4Module` is the second AMQP test. The first, `QuickstartCerts`, runs immediately after `set-modules` and acts as an unintended buffer while the L4 reconcile completes in the background. Whether RouteMessage passes comes down to a race between that buffer and the L4 reconcile/edgeHub-restart, both of which vary run to run depending on the test agents' state (for example how long modules take to reach the `running` state, which depends on image-cache warmth). This is why the test is timing-sensitive rather than deterministically broken. ## Is this masking a 1.6 product regression? No. This was the first thing we checked, since the test began failing around the 1.6 timeframe and a test-only fix would be wrong if it hid a product regression. The decisive evidence is a controlled comparison of the **same binary and commit** run at different times: | Test Run | Date | Commit / CI artifact | Pre-RouteMessage buffer | RouteMessage | |---|---|---|---|---| | [20260529.1](https://msazure.visualstudio.com/One/_build/results?buildId=166045060&view=results) | 2026-05-29 | `b3531a737` / `20260528.1` | 78.6s | **pass**, 21.7s | | [20260710.12](https://msazure.visualstudio.com/One/_build/results?buildId=171795768&view=results) | 2026-07-10 | `b3531a737` / `20260528.1` | 33.1s | **timeout**, 300s | Identical bits, identical commit, identical parameters, clean identity namespace: it passed in May and times out today. A binary cannot regress against itself, so the change is environmental/timing (the buffer shrank relative to the reconcile), not a code regression in 1.6. Supporting data across a range of runs (all AMQP full-suite, on `main` unless noted): | Build | Date | Commit / CI artifact | Buffer | RouteMessage result | |---|---|---|---|---| | [161293998](https://msazure.visualstudio.com/One/_build/results?buildId=161293998&view=results) | 04-22 | `621ff98b` / `20260420.1` | 30.4s | pass 29.5s | | [162168027](https://msazure.visualstudio.com/One/_build/results?buildId=162168027&view=results) | 04-29 | `621ff98b` / `20260427.1` | 30.3s | pass 29.5s | | [166045060](https://msazure.visualstudio.com/One/_build/results?buildId=166045060&view=results) | 05-29 | `b3531a737` / `20260528.1` | 78.6s | pass 21.7s | | [170483773](https://msazure.visualstudio.com/One/_build/results?buildId=170483773&view=results) | 07-01 | `36d6c426` / `20260629.1` | 48.1s | timeout 300s | | [171767938](https://msazure.visualstudio.com/One/_build/results?buildId=171767938&view=results) | 07-10 | `36d6c426` / `20260706.1` | 58.2s | timeout 300s | | [171795768](https://msazure.visualstudio.com/One/_build/results?buildId=171795768&view=results) | 07-10 | `b3531a737` / `20260528.1` | 33.1s | timeout 300s | The outcome does not track a clean buffer threshold (April passed with a 30s buffer; a July run timed out with a 58s buffer), which is expected for a race where both the buffer and the concurrent reconcile time vary. The point the table establishes is narrower and solid: a known-good pre-1.6 binary that passed now times out, so the recent failures are timing/environmental, not a 1.6 product regression. The guard removes the dependency on that accidental buffer entirely by waiting for the actual reconcile, so it is the correct fix regardless of the exact timing. > Some earlier runs (for example [166823784](https://msazure.visualstudio.com/One/_build/results?buildId=166823784&view=results), 06-04) failed fast on `DeviceAlreadyExists` from a leaf identity orphaned by a prior aborted run, a separate issue addressed by the idempotent-identity change below, not the timeout this PR targets. ### Separate, out-of-scope product improvement A ~15s parent restart can strand an in-flight message for minutes because the edgeHub SDK connection-status callback is disabled and edgeHub falls back to slow connectivity polling. That is a genuine product improvement, tracked separately as Fix B in [#7527](https://github.com/Azure/iotedge/pull/7527), and is out of scope here. This PR fixes the test race that makes the failure reproduce in CI. ## Changes 1. **Reconcile-wait guard (the fix).** Adds a guard to the shared `nested-deploy-config.yaml` template, applied to L5 then L4. After `set-modules`, it reads the desired edgeAgent `$version` and polls `reported.lastDesiredVersion` plus `lastDesiredStatus.code`, continuing only when the exact desired version reports status `200`. On a 300-second timeout it prints `lastDesiredStatus.description` and reported module states so the two failure modes (deploy never picked up vs picked up but reported non-200) are distinguishable from the pipeline log. 2. **Collect L4/L5 support bundles.** Adds `Collect_Nested_Bundles` to publish support bundles and raw module logs from L3, L4, and L5. The existing pipeline only preserved L3 evidence, which is why the parent (L4) edgeHub restart was invisible in earlier investigations. Kept because per-level evidence is broadly useful for nested-edge test debugging. 3. **Idempotent leaf-identity creation.** Deletes and recreates a leaf device identity orphaned when a prior run dies before teardown (the pipeline's identity cleanup only removes the L4/L5 parent devices, not the test-created leaf). Prevents a spurious `DeviceAlreadyExistsException` on the shared hub. ## Validation | Build | Harness | RouteMessage | |---|---|---| | [171795530](https://msazure.visualstudio.com/One/_build/results?buildId=171795530&view=results) | with guard | pass 26.8s (full AMQP suite 25/25) | | [171618229](https://msazure.visualstudio.com/One/_build/results?buildId=171618229&view=results) | with guard | pass 21.7s (full AMQP suite 25/25) | | [171795768](https://msazure.visualstudio.com/One/_build/results?buildId=171795768&view=results) | no guard | timeout 300s | With the guard, RouteMessage runs only after the L4/L5 deployment has reconciled, so the edgeHub restart no longer lands mid-test. --- builds/e2e/nested-e2e.yaml | 44 +++++++++++++ .../e2e/templates/nested-deploy-config.yaml | 61 +++++++++++++++++++ .../IotHub.cs | 14 ++++- 3 files changed, 118 insertions(+), 1 deletion(-) diff --git a/builds/e2e/nested-e2e.yaml b/builds/e2e/nested-e2e.yaml index eb82081e697..0466f2a8325 100644 --- a/builds/e2e/nested-e2e.yaml +++ b/builds/e2e/nested-e2e.yaml @@ -152,8 +152,52 @@ stages: - LockAgents - RunNestedTests jobs: + # Capture support bundles from the upper nested levels (L4/L5) after the test stage, + # so the L4/L5 edgeHub/relayer logs are available to debug nested routing failures + # (for example RouteMessageL3LeafToL4Module). The existing per-test teardown already + # publishes L3, so L3 is intentionally omitted here. Runs before Clean_images so + # containers/logs still exist. + - job: Collect_Nested_Bundles + displayName: Collect nested support bundles + condition: always() + strategy: + matrix: + L4: + level: 4 + L5: + level: 5 + pool: + name: $(pool.name) + demands: + - agent-group -equals $(agent.group) + - Agent.OS -equals Linux + - Agent.OSArchitecture -equals X64 + - status -equals locked_$(Build.BuildId)_L$(level) + steps: + - bash: | + set +e + out="$(Build.ArtifactStagingDirectory)/nested-bundle-L$(level)" + mkdir -p "$out" + echo "Collecting support bundle on L$(level) ($(hostname))" + sudo iotedge support-bundle --output "$out/support_bundle_L$(level).zip" || echo "support-bundle failed on L$(level)" + # Also grab raw module logs as a fallback in case support-bundle errors + for m in edgeHub edgeAgent relayer1; do + sudo docker logs "$m" > "$out/$m-L$(level).log" 2>&1 || true + done + ls -la "$out" || true + displayName: 'Generate support bundle (L$(level))' + condition: always() + - task: PublishBuildArtifacts@1 + displayName: 'Publish nested bundle (L$(level))' + condition: always() + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/nested-bundle-L$(level)' + ArtifactName: 'nested-support-bundles' + - job: Clean_images displayName: Clean up Docker images + dependsOn: Collect_Nested_Bundles + condition: always() strategy: matrix: L3: diff --git a/builds/e2e/templates/nested-deploy-config.yaml b/builds/e2e/templates/nested-deploy-config.yaml index f31c37be9c2..5c18e842a0f 100644 --- a/builds/e2e/templates/nested-deploy-config.yaml +++ b/builds/e2e/templates/nested-deploy-config.yaml @@ -47,3 +47,64 @@ steps: # 5/22/2024 - Temporary work around the issue where the az cli command cannot authorize itself within *.sh script using the service principal's service connection deployment_working_file="$(Agent.HomeDirectory)/../working/deployment.json" az iot edge set-modules --auth-type login --device-id "${{ parameters.deviceId }}" --hub-name "$(iotHubName)" --content ${deployment_working_file} --output none + + # Wait for the device's edgeAgent to finish reconciling this deployment before + # returning. set-modules only writes the desired twin; the device applies it + # asynchronously (which restarts edgeHub). Without this barrier the next test + # group can start while a parent is still reconciling, dropping a leaf's upstream + # link mid-test (see RouteMessageL3LeafToL4Module ~4min stall). Gate on edgeAgent's + # reported lastDesiredStatus.code == 200 (Success) for the desired version. + deviceId="${{ parameters.deviceId }}" + hubName="$(iotHubName)" + desiredVersion=$(az iot hub module-twin show --auth-type login --device-id "$deviceId" --module-id '$edgeAgent' --hub-name "$hubName" --query 'properties.desired."$version"' --output tsv) + echo "Waiting for edgeAgent on $deviceId to reconcile desired version $desiredVersion ..." + reconcileTimeoutSecs=300 + reconcilePollSecs=5 + elapsed=0 + while true; do + # Single twin read per poll: az emits the array query as three lines (one per + # element) with --output tsv, so read them into an array. Fewer az invocations + # than one call per field = fewer potential failure points per iteration. + mapfile -t twin < <( + az iot hub module-twin show \ + --auth-type login \ + --device-id "$deviceId" \ + --module-id '$edgeAgent' \ + --hub-name "$hubName" \ + --query '[properties.reported.lastDesiredVersion, properties.reported.lastDesiredStatus.code, properties.reported.lastDesiredStatus.description]' \ + --output tsv \ + 2>/dev/null + ) + reportedVersion="${twin[0]}" + reportedCode="${twin[1]}" + reportedDescription="${twin[2]}" + echo " [$elapsed s] reported version=$reportedVersion status=$reportedCode (want version=$desiredVersion status=200)" + if [ "$reportedVersion" = "$desiredVersion" ] && [ "$reportedCode" = "200" ]; then + echo "edgeAgent on $deviceId reconciled desired version $desiredVersion successfully." + break + fi + if [ "$elapsed" -ge "$reconcileTimeoutSecs" ]; then + # Two distinct failure modes, both diagnosable from the values already read above: + # - reportedVersion never reached desiredVersion => edgeAgent never picked up the + # deployment (twin write lost, agent down, or hub connectivity issue). + # - reportedVersion matched but code stayed non-200 => deployment applied but a + # module is unhealthy (bad image ref, crash loop, pull failure). + echo "----- edgeAgent reconcile timeout diagnostics for $deviceId -----" + echo "desired version : $desiredVersion" + echo "reported version: $reportedVersion" + echo "reported status : $reportedCode" + echo "status detail : $reportedDescription" + if [ "$reportedVersion" != "$desiredVersion" ]; then + echo "diagnosis : edgeAgent never picked up desired version $desiredVersion (still on $reportedVersion) - check agent connectivity/liveness" + else + echo "diagnosis : deployment applied (version matched) but stuck at status $reportedCode - a module is likely unhealthy (bad image, crash loop, or pull failure)" + fi + echo "reported module states:" + az iot hub module-twin show --auth-type login --device-id "$deviceId" --module-id '$edgeAgent' --hub-name "$hubName" --query 'properties.reported.modules' --output json 2>/dev/null || true + echo "----------------------------------------------------------------" + echo "##vso[task.logissue type=error]edgeAgent on $deviceId did not reconcile desired version $desiredVersion within ${reconcileTimeoutSecs}s (last reported version=$reportedVersion status=$reportedCode: $reportedDescription)" + exit 1 + fi + sleep "$reconcilePollSecs" + elapsed=$((elapsed + reconcilePollSecs)) + done diff --git a/test/Microsoft.Azure.Devices.Edge.Test.Common/IotHub.cs b/test/Microsoft.Azure.Devices.Edge.Test.Common/IotHub.cs index 53076824dcf..acb23d7cea1 100644 --- a/test/Microsoft.Azure.Devices.Edge.Test.Common/IotHub.cs +++ b/test/Microsoft.Azure.Devices.Edge.Test.Common/IotHub.cs @@ -112,7 +112,19 @@ public Task GetDeviceIdentityAsync(string deviceId, CancellationToken to public async Task CreateDeviceIdentityAsync(Device device, CancellationToken token) { - return await this.RegistryManager.AddDeviceAsync(device, token); + try + { + return await this.RegistryManager.AddDeviceAsync(device, token); + } + catch (DeviceAlreadyExistsException) + { + // A prior test run can leave an orphaned identity behind (for example when the + // job is killed during artifact upload before cleanup runs). Remove the stale + // identity and recreate so the repro run isn't blocked by leftover state. + Log.Warning($"Device identity '{device.Id}' already exists; deleting orphaned identity and recreating."); + await this.RegistryManager.RemoveDeviceAsync(device.Id, token); + return await this.RegistryManager.AddDeviceAsync(device, token); + } } public async Task CreateEdgeDeviceIdentityAsync(string deviceId, Option parentDeviceId, AuthenticationType authType, X509Thumbprint x509Thumbprint, CancellationToken token) From 1e18481dbc6c1dcbff1576d9960e582837ec3f28 Mon Sep 17 00:00:00 2001 From: Damon Barry Date: Thu, 16 Jul 2026 09:24:59 -0700 Subject: [PATCH 5/6] Increase cleanup timeouts in nested end-to-end pipeline (#7531) The cleanup stage of the nested end-to-end tests has begun failing recently, even though all the jobs within the stage pass. This is because the time it takes to complete the "Clean up identities" job has increased to right around 2 minutes, which is the timeout value for the job. So the job passes, but orchestration at the stage level times out. This change updates the 2 minute timeouts to 5 minutes, which should provide ample time to complete the work. ## Azure IoT Edge PR checklist: --- builds/e2e/nested-e2e.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/builds/e2e/nested-e2e.yaml b/builds/e2e/nested-e2e.yaml index 0466f2a8325..817d7087bca 100644 --- a/builds/e2e/nested-e2e.yaml +++ b/builds/e2e/nested-e2e.yaml @@ -220,7 +220,7 @@ stages: displayName: Unlock agents dependsOn: Clean_images condition: always() - timeoutInMinutes: 2 + timeoutInMinutes: 5 pool: name: $(pool.name) demands: @@ -235,7 +235,7 @@ stages: displayName: Clean up identities dependsOn: Clean_images condition: always() - timeoutInMinutes: 2 + timeoutInMinutes: 5 variables: deviceLvl5DeviceId: $[ stageDependencies.RunNestedTests.SetupVM_level5_mqtt.outputs['createIdentity.parentDeviceId'] ] deviceLvl4DeviceId: $[ stageDependencies.RunNestedTests.SetupVM_level4_mqtt.outputs['createIdentity.parentDeviceId'] ] From c2a2e1b50c7238e0edc62fbb3aaf6bd514f2e877 Mon Sep 17 00:00:00 2001 From: Damon Barry Date: Mon, 20 Jul 2026 10:58:16 -0700 Subject: [PATCH 6/6] Bump Azure Linux build's dependency on Go --- edgelet/build/linux/package-mariner.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/edgelet/build/linux/package-mariner.sh b/edgelet/build/linux/package-mariner.sh index c43afb8a7bc..88ed7028676 100755 --- a/edgelet/build/linux/package-mariner.sh +++ b/edgelet/build/linux/package-mariner.sh @@ -30,8 +30,8 @@ apt-get install -y \ llvm-dev make pigz pkg-config python3-distutils python3-pip qemu-utils rpm tar \ wget zstd -# Install Go 1.23 -GO_VERSION=1.23.0 +# Install Go 1.24 +GO_VERSION=1.24.0 [ "$ARCH" == 'aarch64' ] && GO_ARCH='arm64' || GO_ARCH='amd64' mkdir -p /usr/local/go curl -sSL "https://go.dev/dl/go${GO_VERSION}.linux-${GO_ARCH}.tar.gz" | tar -C /usr/local -xzf -