diff --git a/builds/e2e/nested-e2e.yaml b/builds/e2e/nested-e2e.yaml index eb82081e697..0466f2a8325 100644 --- a/builds/e2e/nested-e2e.yaml +++ b/builds/e2e/nested-e2e.yaml @@ -152,8 +152,52 @@ stages: - LockAgents - RunNestedTests jobs: + # Capture support bundles from the upper nested levels (L4/L5) after the test stage, + # so the L4/L5 edgeHub/relayer logs are available to debug nested routing failures + # (for example RouteMessageL3LeafToL4Module). The existing per-test teardown already + # publishes L3, so L3 is intentionally omitted here. Runs before Clean_images so + # containers/logs still exist. + - job: Collect_Nested_Bundles + displayName: Collect nested support bundles + condition: always() + strategy: + matrix: + L4: + level: 4 + L5: + level: 5 + pool: + name: $(pool.name) + demands: + - agent-group -equals $(agent.group) + - Agent.OS -equals Linux + - Agent.OSArchitecture -equals X64 + - status -equals locked_$(Build.BuildId)_L$(level) + steps: + - bash: | + set +e + out="$(Build.ArtifactStagingDirectory)/nested-bundle-L$(level)" + mkdir -p "$out" + echo "Collecting support bundle on L$(level) ($(hostname))" + sudo iotedge support-bundle --output "$out/support_bundle_L$(level).zip" || echo "support-bundle failed on L$(level)" + # Also grab raw module logs as a fallback in case support-bundle errors + for m in edgeHub edgeAgent relayer1; do + sudo docker logs "$m" > "$out/$m-L$(level).log" 2>&1 || true + done + ls -la "$out" || true + displayName: 'Generate support bundle (L$(level))' + condition: always() + - task: PublishBuildArtifacts@1 + displayName: 'Publish nested bundle (L$(level))' + condition: always() + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/nested-bundle-L$(level)' + ArtifactName: 'nested-support-bundles' + - job: Clean_images displayName: Clean up Docker images + dependsOn: Collect_Nested_Bundles + condition: always() strategy: matrix: L3: diff --git a/builds/e2e/templates/nested-deploy-config.yaml b/builds/e2e/templates/nested-deploy-config.yaml index f31c37be9c2..5c18e842a0f 100644 --- a/builds/e2e/templates/nested-deploy-config.yaml +++ b/builds/e2e/templates/nested-deploy-config.yaml @@ -47,3 +47,64 @@ steps: # 5/22/2024 - Temporary work around the issue where the az cli command cannot authorize itself within *.sh script using the service principal's service connection deployment_working_file="$(Agent.HomeDirectory)/../working/deployment.json" az iot edge set-modules --auth-type login --device-id "${{ parameters.deviceId }}" --hub-name "$(iotHubName)" --content ${deployment_working_file} --output none + + # Wait for the device's edgeAgent to finish reconciling this deployment before + # returning. set-modules only writes the desired twin; the device applies it + # asynchronously (which restarts edgeHub). Without this barrier the next test + # group can start while a parent is still reconciling, dropping a leaf's upstream + # link mid-test (see RouteMessageL3LeafToL4Module ~4min stall). Gate on edgeAgent's + # reported lastDesiredStatus.code == 200 (Success) for the desired version. + deviceId="${{ parameters.deviceId }}" + hubName="$(iotHubName)" + desiredVersion=$(az iot hub module-twin show --auth-type login --device-id "$deviceId" --module-id '$edgeAgent' --hub-name "$hubName" --query 'properties.desired."$version"' --output tsv) + echo "Waiting for edgeAgent on $deviceId to reconcile desired version $desiredVersion ..." + reconcileTimeoutSecs=300 + reconcilePollSecs=5 + elapsed=0 + while true; do + # Single twin read per poll: az emits the array query as three lines (one per + # element) with --output tsv, so read them into an array. Fewer az invocations + # than one call per field = fewer potential failure points per iteration. + mapfile -t twin < <( + az iot hub module-twin show \ + --auth-type login \ + --device-id "$deviceId" \ + --module-id '$edgeAgent' \ + --hub-name "$hubName" \ + --query '[properties.reported.lastDesiredVersion, properties.reported.lastDesiredStatus.code, properties.reported.lastDesiredStatus.description]' \ + --output tsv \ + 2>/dev/null + ) + reportedVersion="${twin[0]}" + reportedCode="${twin[1]}" + reportedDescription="${twin[2]}" + echo " [$elapsed s] reported version=$reportedVersion status=$reportedCode (want version=$desiredVersion status=200)" + if [ "$reportedVersion" = "$desiredVersion" ] && [ "$reportedCode" = "200" ]; then + echo "edgeAgent on $deviceId reconciled desired version $desiredVersion successfully." + break + fi + if [ "$elapsed" -ge "$reconcileTimeoutSecs" ]; then + # Two distinct failure modes, both diagnosable from the values already read above: + # - reportedVersion never reached desiredVersion => edgeAgent never picked up the + # deployment (twin write lost, agent down, or hub connectivity issue). + # - reportedVersion matched but code stayed non-200 => deployment applied but a + # module is unhealthy (bad image ref, crash loop, pull failure). + echo "----- edgeAgent reconcile timeout diagnostics for $deviceId -----" + echo "desired version : $desiredVersion" + echo "reported version: $reportedVersion" + echo "reported status : $reportedCode" + echo "status detail : $reportedDescription" + if [ "$reportedVersion" != "$desiredVersion" ]; then + echo "diagnosis : edgeAgent never picked up desired version $desiredVersion (still on $reportedVersion) - check agent connectivity/liveness" + else + echo "diagnosis : deployment applied (version matched) but stuck at status $reportedCode - a module is likely unhealthy (bad image, crash loop, or pull failure)" + fi + echo "reported module states:" + az iot hub module-twin show --auth-type login --device-id "$deviceId" --module-id '$edgeAgent' --hub-name "$hubName" --query 'properties.reported.modules' --output json 2>/dev/null || true + echo "----------------------------------------------------------------" + echo "##vso[task.logissue type=error]edgeAgent on $deviceId did not reconcile desired version $desiredVersion within ${reconcileTimeoutSecs}s (last reported version=$reportedVersion status=$reportedCode: $reportedDescription)" + exit 1 + fi + sleep "$reconcilePollSecs" + elapsed=$((elapsed + reconcilePollSecs)) + done diff --git a/test/Microsoft.Azure.Devices.Edge.Test.Common/IotHub.cs b/test/Microsoft.Azure.Devices.Edge.Test.Common/IotHub.cs index 80b02d384ac..ec01cd88c1a 100644 --- a/test/Microsoft.Azure.Devices.Edge.Test.Common/IotHub.cs +++ b/test/Microsoft.Azure.Devices.Edge.Test.Common/IotHub.cs @@ -97,7 +97,19 @@ public Task GetDeviceIdentityAsync(string deviceId, CancellationToken to public async Task CreateDeviceIdentityAsync(Device device, CancellationToken token) { - return await this.RegistryManager.AddDeviceAsync(device, token); + try + { + return await this.RegistryManager.AddDeviceAsync(device, token); + } + catch (DeviceAlreadyExistsException) + { + // A prior test run can leave an orphaned identity behind (for example when the + // job is killed during artifact upload before cleanup runs). Remove the stale + // identity and recreate so the repro run isn't blocked by leftover state. + Log.Warning($"Device identity '{device.Id}' already exists; deleting orphaned identity and recreating."); + await this.RegistryManager.RemoveDeviceAsync(device.Id, token); + return await this.RegistryManager.AddDeviceAsync(device, token); + } } public async Task CreateEdgeDeviceIdentityAsync(string deviceId, Option parentDeviceId, AuthenticationType authType, X509Thumbprint x509Thumbprint, CancellationToken token)