diff --git a/builds/e2e/nested-e2e.yaml b/builds/e2e/nested-e2e.yaml index 0ac4ebe9309..3ffd31790de 100644 --- a/builds/e2e/nested-e2e.yaml +++ b/builds/e2e/nested-e2e.yaml @@ -152,8 +152,52 @@ stages: - LockAgents - RunNestedTests jobs: + # TEMP (investigation, not for merge): capture support bundles from ALL levels (L3/L4/L5) + # after the test stage, so the L4/L5 edgeHub/relayer logs are available to debug + # RouteMessageL3LeafToL4Module. Runs before Clean_images so containers/logs still exist. + - job: Collect_Nested_Bundles + displayName: Collect nested support bundles + condition: always() + strategy: + matrix: + L3: + level: 3 + L4: + level: 4 + L5: + level: 5 + pool: + name: $(pool.name) + demands: + - agent-group -equals $(agent.group) + - Agent.OS -equals Linux + - Agent.OSArchitecture -equals X64 + - status -equals locked_$(Build.BuildId)_L$(level) + steps: + - bash: | + set +e + out="$(Build.ArtifactStagingDirectory)/nested-bundle-L$(level)" + mkdir -p "$out" + echo "Collecting support bundle on L$(level) ($(hostname))" + sudo iotedge support-bundle --output "$out/support_bundle_L$(level).zip" || echo "support-bundle failed on L$(level)" + # Also grab raw module logs as a fallback in case support-bundle errors + for m in edgeHub edgeAgent relayer1; do + sudo docker logs "$m" > "$out/$m-L$(level).log" 2>&1 || true + done + ls -la "$out" || true + displayName: 'Generate support bundle (L$(level))' + condition: always() + - task: PublishBuildArtifacts@1 + displayName: 'Publish nested bundle (L$(level))' + condition: always() + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/nested-bundle-L$(level)' + ArtifactName: 'nested-support-bundles' + - job: Clean_images displayName: Clean up Docker images + dependsOn: Collect_Nested_Bundles + condition: always() strategy: matrix: L3: diff --git a/e2e_deployment_files/nestededge_middleLayerBaseDeployment_amqp.json b/e2e_deployment_files/nestededge_middleLayerBaseDeployment_amqp.json index b8fd9b166d1..b8dd13b55af 100644 --- a/e2e_deployment_files/nestededge_middleLayerBaseDeployment_amqp.json +++ b/e2e_deployment_files/nestededge_middleLayerBaseDeployment_amqp.json @@ -27,9 +27,12 @@ "type": "docker", "settings": { "image": "$upstream:443/microsoft/azureiotedge-hub:-linux-", - "createOptions": "{\"HostConfig\": {\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"4m\",\"max-file\":\"7\",\"compress\":\"true\"}}, \"PortBindings\": {\"8883/tcp\": [{\"HostPort\": \"8883\"}],\"5671/tcp\": [{\"HostPort\": \"5671\"}]}}}" + "createOptions": "{\"HostConfig\": {\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"4m\",\"max-file\":\"7\",\"compress\":\"true\"}}, \"PortBindings\": {\"8883/tcp\": [{\"HostPort\": \"8883\"}],\"5671/tcp\": [{\"HostPort\": \"5671\"}]}, \"Binds\": [\"/etc/iotedge/storage/:/iotedge/storage/\"]}}" }, "env": { + "StorageFolder": { + "value": "/iotedge/storage" + }, "DeviceScopeCacheRefreshDelaySecs": { "value": 0 }, diff --git a/e2e_deployment_files/nestededge_middleLayer_e2e_amqp.json b/e2e_deployment_files/nestededge_middleLayer_e2e_amqp.json index 8b8900ca14e..ab9b763dc3e 100644 --- a/e2e_deployment_files/nestededge_middleLayer_e2e_amqp.json +++ b/e2e_deployment_files/nestededge_middleLayer_e2e_amqp.json @@ -27,9 +27,12 @@ "type": "docker", "settings": { "image": "$upstream:443/microsoft/azureiotedge-hub:-linux-", - "createOptions": "{\"HostConfig\": {\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"4m\",\"max-file\":\"7\",\"compress\":\"true\"}}, \"PortBindings\": {\"8883/tcp\": [{\"HostPort\": \"8883\"}],\"5671/tcp\": [{\"HostPort\": \"5671\"}]}}}" + "createOptions": "{\"HostConfig\": {\"LogConfig\":{\"Type\":\"json-file\",\"Config\":{\"max-size\":\"4m\",\"max-file\":\"7\",\"compress\":\"true\"}}, \"PortBindings\": {\"8883/tcp\": [{\"HostPort\": \"8883\"}],\"5671/tcp\": [{\"HostPort\": \"5671\"}]}, \"Binds\": [\"/etc/iotedge/storage/:/iotedge/storage/\"]}}" }, "env": { + "StorageFolder": { + "value": "/iotedge/storage" + }, "experimentalFeatures__enabled": { "value": "true" }, diff --git a/test/Microsoft.Azure.Devices.Edge.Test.Common/IotHub.cs b/test/Microsoft.Azure.Devices.Edge.Test.Common/IotHub.cs index 2f811a4e95b..7dca22a98ab 100644 --- a/test/Microsoft.Azure.Devices.Edge.Test.Common/IotHub.cs +++ b/test/Microsoft.Azure.Devices.Edge.Test.Common/IotHub.cs @@ -97,7 +97,19 @@ public Task GetDeviceIdentityAsync(string deviceId, CancellationToken to public async Task CreateDeviceIdentityAsync(Device device, CancellationToken token) { - return await this.RegistryManager.AddDeviceAsync(device, token); + try + { + return await this.RegistryManager.AddDeviceAsync(device, token); + } + catch (DeviceAlreadyExistsException) + { + // A prior test run can leave an orphaned identity behind (for example when the + // job is killed during artifact upload before cleanup runs). Remove the stale + // identity and recreate so the repro run isn't blocked by leftover state. + Log.Warning($"Device identity '{device.Id}' already exists; deleting orphaned identity and recreating."); + await this.RegistryManager.RemoveDeviceAsync(device.Id, token); + return await this.RegistryManager.AddDeviceAsync(device, token); + } } public async Task CreateEdgeDeviceIdentityAsync(string deviceId, Option parentDeviceId, AuthenticationType authType, X509Thumbprint x509Thumbprint, CancellationToken token)