From f0ef7759c78951c9f70880e02d2836e450f9f8e4 Mon Sep 17 00:00:00 2001 From: John Lian Date: Tue, 7 Jul 2026 15:59:50 -0700 Subject: [PATCH 1/4] [TEMP investigation] Nested E2E: minimal RouteMessage repro + collect L4/L5 support bundles Not for merge. Two temporary changes to investigate the consistently-failing RouteMessageL3LeafToL4Module nested-e2e test (AMQP-only): 1. e2e-run.yaml: narrow the nestededge_amqp filter to the smallest set that reproduces the failure (Module + Metrics + IoTEdgeCheck + RouteMessage), derived from PR #7521's bisection (run 20260701.8 first-failing subset). 2. nested-e2e.yaml: add a Collect_Nested_Bundles job that captures support bundles + raw module logs from ALL levels (L3/L4/L5) after the test stage. The default pipeline only collects the L3 bundle, so the L4 relayer/edgeHub logs that explain the 4-minute store-and-forward stall are never captured. --- builds/e2e/nested-e2e.yaml | 44 +++++++++++++++++++++++++++++++ builds/e2e/templates/e2e-run.yaml | 4 +++ 2 files changed, 48 insertions(+) diff --git a/builds/e2e/nested-e2e.yaml b/builds/e2e/nested-e2e.yaml index 0ac4ebe9309..3ffd31790de 100644 --- a/builds/e2e/nested-e2e.yaml +++ b/builds/e2e/nested-e2e.yaml @@ -152,8 +152,52 @@ stages: - LockAgents - RunNestedTests jobs: + # TEMP (investigation, not for merge): capture support bundles from ALL levels (L3/L4/L5) + # after the test stage, so the L4/L5 edgeHub/relayer logs are available to debug + # RouteMessageL3LeafToL4Module. Runs before Clean_images so containers/logs still exist. + - job: Collect_Nested_Bundles + displayName: Collect nested support bundles + condition: always() + strategy: + matrix: + L3: + level: 3 + L4: + level: 4 + L5: + level: 5 + pool: + name: $(pool.name) + demands: + - agent-group -equals $(agent.group) + - Agent.OS -equals Linux + - Agent.OSArchitecture -equals X64 + - status -equals locked_$(Build.BuildId)_L$(level) + steps: + - bash: | + set +e + out="$(Build.ArtifactStagingDirectory)/nested-bundle-L$(level)" + mkdir -p "$out" + echo "Collecting support bundle on L$(level) ($(hostname))" + sudo iotedge support-bundle --output "$out/support_bundle_L$(level).zip" || echo "support-bundle failed on L$(level)" + # Also grab raw module logs as a fallback in case support-bundle errors + for m in edgeHub edgeAgent relayer1; do + sudo docker logs "$m" > "$out/$m-L$(level).log" 2>&1 || true + done + ls -la "$out" || true + displayName: 'Generate support bundle (L$(level))' + condition: always() + - task: PublishBuildArtifacts@1 + displayName: 'Publish nested bundle (L$(level))' + condition: always() + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/nested-bundle-L$(level)' + ArtifactName: 'nested-support-bundles' + - job: Clean_images displayName: Clean up Docker images + dependsOn: Collect_Nested_Bundles + condition: always() strategy: matrix: L3: diff --git a/builds/e2e/templates/e2e-run.yaml b/builds/e2e/templates/e2e-run.yaml index 7426f3f347c..611b7787508 100644 --- a/builds/e2e/templates/e2e-run.yaml +++ b/builds/e2e/templates/e2e-run.yaml @@ -67,6 +67,10 @@ steps: # Below tests were disabled and marked for re-enable when a blocking item was resolved. # When it was resolved the tests were never enabled. We need to re-enable these. $filter += '&FullyQualifiedName!~Provisioning&FullyQualifiedName!~SasOutOfScope&FullyQualifiedName!~X509ManualProvision&FullyQualifiedName!~AuthorizationPolicyUpdateTest&FullyQualifiedName!~AuthorizationPolicyExplicitPolicyTest' + # TEMP (investigation, not for merge): narrow to the smallest set that reproduces + # RouteMessageL3LeafToL4Module. Metrics + IoTEdgeCheck must run before RouteMessage + # to provoke the AMQP upstream blip (derived from PR #7521 bisection, run 20260701.8). + $filter += '&(FullyQualifiedName~Microsoft.Azure.Devices.Edge.Test.Module|FullyQualifiedName~Microsoft.Azure.Devices.Edge.Test.Metrics|FullyQualifiedName~IoTEdgeCheck|FullyQualifiedName~RouteMessageL3LeafToL4Module)' } elseif ($test_type -eq 'nestededge_isa95') { From 141abe2179aa35ddf4a188572d2975fdb10cb41a Mon Sep 17 00:00:00 2001 From: John Lian Date: Wed, 8 Jul 2026 12:16:36 -0700 Subject: [PATCH 2/4] Remove test filter to run full unperturbed suite (keep only L4/L5 bundle collection) The temporary category filter narrowed and reordered the nested suite, causing RouteMessage to run first before relayer1 was deployed on L4 (a different failure than the original 15s upstream blip). Revert the filter so the full suite runs with RouteMessage in its normal late position; keep only the Collect_Nested_Bundles job so L4/L5 evidence is captured without perturbing the test flow. --- builds/e2e/templates/e2e-run.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/builds/e2e/templates/e2e-run.yaml b/builds/e2e/templates/e2e-run.yaml index 611b7787508..7426f3f347c 100644 --- a/builds/e2e/templates/e2e-run.yaml +++ b/builds/e2e/templates/e2e-run.yaml @@ -67,10 +67,6 @@ steps: # Below tests were disabled and marked for re-enable when a blocking item was resolved. # When it was resolved the tests were never enabled. We need to re-enable these. $filter += '&FullyQualifiedName!~Provisioning&FullyQualifiedName!~SasOutOfScope&FullyQualifiedName!~X509ManualProvision&FullyQualifiedName!~AuthorizationPolicyUpdateTest&FullyQualifiedName!~AuthorizationPolicyExplicitPolicyTest' - # TEMP (investigation, not for merge): narrow to the smallest set that reproduces - # RouteMessageL3LeafToL4Module. Metrics + IoTEdgeCheck must run before RouteMessage - # to provoke the AMQP upstream blip (derived from PR #7521 bisection, run 20260701.8). - $filter += '&(FullyQualifiedName~Microsoft.Azure.Devices.Edge.Test.Module|FullyQualifiedName~Microsoft.Azure.Devices.Edge.Test.Metrics|FullyQualifiedName~IoTEdgeCheck|FullyQualifiedName~RouteMessageL3LeafToL4Module)' } elseif ($test_type -eq 'nestededge_isa95') { From bb85225addada063b4047529ad5b995f39f941dd Mon Sep 17 00:00:00 2001 From: John Lian Date: Wed, 8 Jul 2026 19:22:01 -0700 Subject: [PATCH 3/4] test: make leaf device identity creation idempotent for nested-e2e repro CreateDeviceIdentityAsync now deletes+recreates on DeviceAlreadyExists so an orphaned identity from a prior killed run doesn't block the RouteMessage repro. --- .../IotHub.cs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/test/Microsoft.Azure.Devices.Edge.Test.Common/IotHub.cs b/test/Microsoft.Azure.Devices.Edge.Test.Common/IotHub.cs index 2f811a4e95b..7dca22a98ab 100644 --- a/test/Microsoft.Azure.Devices.Edge.Test.Common/IotHub.cs +++ b/test/Microsoft.Azure.Devices.Edge.Test.Common/IotHub.cs @@ -97,7 +97,19 @@ public Task GetDeviceIdentityAsync(string deviceId, CancellationToken to public async Task CreateDeviceIdentityAsync(Device device, CancellationToken token) { - return await this.RegistryManager.AddDeviceAsync(device, token); + try + { + return await this.RegistryManager.AddDeviceAsync(device, token); + } + catch (DeviceAlreadyExistsException) + { + // A prior test run can leave an orphaned identity behind (for example when the + // job is killed during artifact upload before cleanup runs). Remove the stale + // identity and recreate so the repro run isn't blocked by leftover state. + Log.Warning($"Device identity '{device.Id}' already exists; deleting orphaned identity and recreating."); + await this.RegistryManager.RemoveDeviceAsync(device.Id, token); + return await this.RegistryManager.AddDeviceAsync(device, token); + } } public async Task CreateEdgeDeviceIdentityAsync(string deviceId, Option parentDeviceId, AuthenticationType authType, X509Thumbprint x509Thumbprint, CancellationToken token) From f3c70765c32dce91b2bad582f5ce3267b281b6c2 Mon Sep 17 00:00:00 2001 From: John Lian Date: Mon, 13 Jul 2026 15:50:58 -0700 Subject: [PATCH 4/4] test: persist L4 edgeHub store across container recreate (nested-e2e diagnostic) The nested-e2e edgeHub stores store-and-forward messages at the default /tmp/edgeHub, which lives in the container's ephemeral writable layer. When the AMQP-group redeploy recreates the L4 edgeHub container mid-test, any queued/undelivered message is lost with the wiped store. Mount a host directory (/etc/iotedge/storage) and point edgeHub's StorageFolder at it so the message store survives an edgeHub container recreate. edgeHub's entrypoint (hubStart.sh) creates and chowns $StorageFolder/edgeHub to the edgehub uid, so no host-side prep is needed. Diagnostic intent: separate two hypotheses for RouteMessageL3LeafToL4Module. If the test passes with a persistent store, the failing message was being lost to the container-recreate store wipe. If it still stalls, the message is orphaned within a surviving store (the edgeHub store-and-forward re-pump / checkpoint defect that #7528 targets). --- .../nestededge_middleLayerBaseDeployment_amqp.json | 5 ++++- e2e_deployment_files/nestededge_middleLayer_e2e_amqp.json | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/e2e_deployment_files/nestededge_middleLayerBaseDeployment_amqp.json b/e2e_deployment_files/nestededge_middleLayerBaseDeployment_amqp.json index b8fd9b166d1..b8dd13b55af 100644 --- a/e2e_deployment_files/nestededge_middleLayerBaseDeployment_amqp.json +++ b/e2e_deployment_files/nestededge_middleLayerBaseDeployment_amqp.json @@ -27,9 +27,12 @@ "type": "docker", "settings": { "image": "$upstream:443/microsoft/azureiotedge-hub:-linux-", - "createOptions": "{\"HostConfig\": {\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"4m\",\"max-file\":\"7\",\"compress\":\"true\"}}, \"PortBindings\": {\"8883/tcp\": [{\"HostPort\": \"8883\"}],\"5671/tcp\": [{\"HostPort\": \"5671\"}]}}}" + "createOptions": "{\"HostConfig\": {\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"4m\",\"max-file\":\"7\",\"compress\":\"true\"}}, \"PortBindings\": {\"8883/tcp\": [{\"HostPort\": \"8883\"}],\"5671/tcp\": [{\"HostPort\": \"5671\"}]}, \"Binds\": [\"/etc/iotedge/storage/:/iotedge/storage/\"]}}" }, "env": { + "StorageFolder": { + "value": "/iotedge/storage" + }, "DeviceScopeCacheRefreshDelaySecs": { "value": 0 }, diff --git a/e2e_deployment_files/nestededge_middleLayer_e2e_amqp.json b/e2e_deployment_files/nestededge_middleLayer_e2e_amqp.json index 8b8900ca14e..ab9b763dc3e 100644 --- a/e2e_deployment_files/nestededge_middleLayer_e2e_amqp.json +++ b/e2e_deployment_files/nestededge_middleLayer_e2e_amqp.json @@ -27,9 +27,12 @@ "type": "docker", "settings": { "image": "$upstream:443/microsoft/azureiotedge-hub:-linux-", - "createOptions": "{\"HostConfig\": {\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"4m\",\"max-file\":\"7\",\"compress\":\"true\"}}, \"PortBindings\": {\"8883/tcp\": [{\"HostPort\": \"8883\"}],\"5671/tcp\": [{\"HostPort\": \"5671\"}]}}}" + "createOptions": "{\"HostConfig\": {\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"4m\",\"max-file\":\"7\",\"compress\":\"true\"}}, \"PortBindings\": {\"8883/tcp\": [{\"HostPort\": \"8883\"}],\"5671/tcp\": [{\"HostPort\": \"5671\"}]}, \"Binds\": [\"/etc/iotedge/storage/:/iotedge/storage/\"]}}" }, "env": { + "StorageFolder": { + "value": "/iotedge/storage" + }, "experimentalFeatures__enabled": { "value": "true" },