diff --git a/eng/performance/maui_scenarios_ios.proj b/eng/performance/maui_scenarios_ios.proj index 084344d6aa0..0231c4e69c5 100644 --- a/eng/performance/maui_scenarios_ios.proj +++ b/eng/performance/maui_scenarios_ios.proj @@ -2,45 +2,61 @@ - true + <_IsInnerLoop Condition="'$(RunKind)' == 'maui_scenarios_ios_innerloop'">true + true - + $(Python) post.py scenarios_out $(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)\ $(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)/ - + <_MSBuildArgs Condition="'$(RuntimeFlavor)' == 'mono'">/p:UseMonoRuntime=true;/p:_DisableCheckForUnsupportedMonoMobileRuntime=true <_MSBuildArgs Condition="'$(RuntimeFlavor)' == 'coreclr'">/p:UseMonoRuntime=false - - - - <_MSBuildArgs Condition="'$(RuntimeFlavor)' == 'coreclr'">$(_MSBuildArgs);/p:PublishReadyToRunStripDebugInfo=false;/p:PublishReadyToRunStripInliningInfo=false - - <_MSBuildArgs Condition="'$(RuntimeFlavor)' == 'coreclr' and '$(CodegenType)' == 'Interpreter'">$(_MSBuildArgs);/p:PublishReadyToRun=false - <_MSBuildArgs Condition="'$(RuntimeFlavor)' == 'coreclr' and '$(CodegenType)' == 'NativeAOT'">$(_MSBuildArgs);/p:PublishAot=true;/p:PublishAotUsingRuntimePack=true + + + + <_MSBuildArgs Condition="'$(RuntimeFlavor)' == 'mono'">/p:UseMonoRuntime=true /p:_DisableCheckForUnsupportedMonoMobileRuntime=true + <_MSBuildArgs Condition="'$(RuntimeFlavor)' == 'coreclr'">/p:UseMonoRuntime=false + iossimulator-x64 + <_MSBuildArgs>$(_MSBuildArgs) /p:RuntimeIdentifier=$(iOSRid) + <_MSBuildArgs Condition="'$(iOSRid)' == 'ios-arm64'">$(_MSBuildArgs) /p:EnableCodeSigning=false + <_MSBuildArgs>$(_MSBuildArgs) /p:MtouchLink=None + + <_MSBuildArgs>$(_MSBuildArgs) /p:ValidateXcodeVersion=false + $(RuntimeFlavor)_Default + 3 + <_MacEnvVars>export DOTNET_ROOT=$HELIX_CORRELATION_PAYLOAD/dotnet;export DOTNET_CLI_TELEMETRY_OPTOUT=1;export DOTNET_MULTILEVEL_LOOKUP=0;export NUGET_PACKAGES=$HELIX_WORKITEM_ROOT/.packages;export PATH=$HELIX_CORRELATION_PAYLOAD/dotnet:$PATH - - - + + - 00:30 + 00:30 + 02:30 - + netios $(ScenariosDir)%(ScenarioDirectoryName) @@ -67,7 +83,7 @@ - + XCODE_PATH=`find /Applications -maxdepth 1 -type d -name 'Xcode_*.app' | sort -t_ -k2 -V | tail -1` && echo "Selected Xcode: $XCODE_PATH" && sudo xcode-select -s "$XCODE_PATH" && $(Python) pre.py publish -f $(PERFLAB_Framework)-ios --self-contained -c $(BuildConfig) -r ios-arm64 --msbuild="$(_MSBuildArgs)" --binlog $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName)/%(PreparePayloadWorkItem.ScenarioDirectoryName).binlog -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName) && rm -rf app/obj app/bin && cd ../ && zip -r %(PreparePayloadWorkItem.ScenarioDirectoryName).zip %(PreparePayloadWorkItem.ScenarioDirectoryName) @@ -75,7 +91,7 @@ - + cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName) $HELIX_WORKITEM_ROOT/pub $(Python) test.py sod --scenario-name "%(Identity)" $(ScenarioArgs) @@ -124,7 +140,7 @@ - - + + + mauiiosinnerloop + $(ScenariosDir)%(ScenarioDirectoryName) + + + + + + XCODE_PATH=`find /Applications -maxdepth 1 -type d -name 'Xcode_*.app' | sort -t_ -k2 -V | tail -1` && echo "Selected Xcode: $XCODE_PATH" && sudo xcode-select -s "$XCODE_PATH" && $(Python) pre.py default -f $(PERFLAB_Framework)-ios + %(PreparePayloadWorkItem.PayloadDirectory) + + + + + + $(_MacEnvVars);export IOS_RID=$(iOSRid);$(Python) setup_helix.py $(PERFLAB_Framework)-ios "$(_MSBuildArgs)" || exit $? + $(Python) test.py iosinnerloop --csproj-path app/MauiiOSInnerLoop.csproj --edit-src "src/MainPage.xaml.cs;src/MainPage.xaml" --edit-dest "app/Pages/MainPage.xaml.cs;app/Pages/MainPage.xaml" --bundle-id net.dot.mauiiosinnerloop -f $(PERFLAB_Framework)-ios -c Debug --msbuild-args "$(_MSBuildArgs)" --device-type simulator --inner-loop-iterations $(InnerLoopIterations) --scenario-name "%(Identity)" $(ScenarioArgs) + export IOS_RID=$(iOSRid);$(Python) post.py + output.log + + + + + + $(_MacEnvVars);export IOS_RID=$(iOSRid);$(Python) setup_helix.py $(PERFLAB_Framework)-ios "$(_MSBuildArgs)" || exit $? + $(Python) test.py iosinnerloop --csproj-path app/MauiiOSInnerLoop.csproj --edit-src "src/MainPage.xaml.cs;src/MainPage.xaml" --edit-dest "app/Pages/MainPage.xaml.cs;app/Pages/MainPage.xaml" --bundle-id net.dot.mauiiosinnerloop -f $(PERFLAB_Framework)-ios -c Debug --msbuild-args "$(_MSBuildArgs)" --device-type device --inner-loop-iterations $(InnerLoopIterations) --scenario-name "%(Identity)" $(ScenarioArgs) + export IOS_RID=$(iOSRid);$(Python) post.py + output.log + + - - + export PYTHONPATH=$ORIGPYPATH;$(HelixPostCommands) diff --git a/eng/pipelines/sdk-perf-jobs.yml b/eng/pipelines/sdk-perf-jobs.yml index 8c9115a8b63..9251a4c375a 100644 --- a/eng/pipelines/sdk-perf-jobs.yml +++ b/eng/pipelines/sdk-perf-jobs.yml @@ -17,7 +17,7 @@ jobs: # Public correctness jobs ###################################################### -- ${{ if parameters.runPublicJobs }}: +- ${{ if false }}: # parameters.runPublicJobs — disabled for iOS Inner Loop WIP - job: Tooling_Tests_Windows displayName: Tooling Tests (Windows) timeoutInMinutes: 30 @@ -54,7 +54,7 @@ jobs: dotnet test src\tools\ScenarioMeasurement\Startup.Tests\Startup.Tests.csproj --configuration Release --nologo --verbosity minimal displayName: Run tooling tests -- ${{ if parameters.runPublicJobs }}: +- ${{ if false }}: # parameters.runPublicJobs — disabled for iOS Inner Loop WIP # Scenario benchmarks - template: /eng/pipelines/templates/build-machine-matrix.yml @@ -364,243 +364,416 @@ jobs: - ${{ if parameters.runPrivateJobs }}: # Scenario benchmarks - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - win-x64-viper - - ubuntu-x64-viper - - win-arm64-ampere - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - runKind: scenarios - projectFileName: scenarios.proj - channels: - - main - - 9.0 - - 8.0 - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + - ${{ if false }}: # disabled: iOS-only WIP test run + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-viper + - ubuntu-x64-viper + - win-arm64-ampere + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + runKind: scenarios + projectFileName: scenarios.proj + channels: + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} # Affinitized Scenario benchmarks (Initially just PDN) - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - win-x64-viper - - win-arm64-ampere - isPublic: false - jobParameters: - runKind: scenarios - projectFileName: scenarios_affinitized.proj - channels: - - main - - 9.0 - - 8.0 - additionalJobIdentifier: 'Affinity_85' - affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account - runEnvVars: - - DOTNET_GCgen0size=410000 # ~4MB - - DOTNET_GCHeapCount=4 - - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + - ${{ if false }}: # disabled: iOS-only WIP test run + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-viper + - win-arm64-ampere + isPublic: false + jobParameters: + runKind: scenarios + projectFileName: scenarios_affinitized.proj + channels: + - main + - 9.0 + - 8.0 + additionalJobIdentifier: 'Affinity_85' + affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account + runEnvVars: + - DOTNET_GCgen0size=410000 # ~4MB + - DOTNET_GCHeapCount=4 + - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Maui Android scenario benchmarks (Mono Default) - Release + - ${{ if false }}: # disabled: iOS-only WIP test run + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-android-arm64-pixel + - win-x64-android-arm64-galaxy + isPublic: false + jobParameters: + runKind: maui_scenarios_android + projectFileName: maui_scenarios_android.proj + channels: + - main + runtimeFlavor: mono + codeGenType: Default + buildConfig: Release + additionalJobIdentifier: Mono + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} # Maui Android scenario benchmarks (CoreCLR Default) - Release - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - win-x64-android-arm64-pixel - - win-x64-android-arm64-galaxy - isPublic: false - jobParameters: - runKind: maui_scenarios_android - projectFileName: maui_scenarios_android.proj - channels: - - main - runtimeFlavor: coreclr - codeGenType: Default - buildConfig: Release - additionalJobIdentifier: CoreCLR - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + - ${{ if false }}: # disabled: iOS-only WIP test run + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-android-arm64-pixel + - win-x64-android-arm64-galaxy + isPublic: false + jobParameters: + runKind: maui_scenarios_android + projectFileName: maui_scenarios_android.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: Default + buildConfig: Release + additionalJobIdentifier: CoreCLR + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} # Maui Android scenario benchmarks (CoreCLR NativeAOT) - Release - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - win-x64-android-arm64-pixel - - win-x64-android-arm64-galaxy - isPublic: false - jobParameters: - runKind: maui_scenarios_android - projectFileName: maui_scenarios_android.proj - channels: - - main - runtimeFlavor: coreclr - codeGenType: NativeAOT - buildConfig: Release - additionalJobIdentifier: CoreCLR - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + - ${{ if false }}: # disabled: iOS-only WIP test run + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-android-arm64-pixel + - win-x64-android-arm64-galaxy + isPublic: false + jobParameters: + runKind: maui_scenarios_android + projectFileName: maui_scenarios_android.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: NativeAOT + buildConfig: Release + additionalJobIdentifier: CoreCLR + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} - # Maui Android scenario benchmarks (Mono Default) - Release - # Smoke coverage that the MAUI Android app still builds and runs on Mono. - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - win-x64-android-arm64-pixel - - win-x64-android-arm64-galaxy - isPublic: false - jobParameters: - runKind: maui_scenarios_android - projectFileName: maui_scenarios_android.proj - channels: - - main - runtimeFlavor: mono - codeGenType: Default - buildConfig: Release - additionalJobIdentifier: Mono - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + - ${{ if false }}: # disabled for iOS inner-loop WIP; re-enable before merge + # Maui iOS scenario benchmarks (Mono Default) - Release + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - osx-x64-ios-arm64 + isPublic: false + jobParameters: + runKind: maui_scenarios_ios + projectFileName: maui_scenarios_ios.proj + channels: + - main + runtimeFlavor: mono + codeGenType: Default + buildConfig: Release + additionalJobIdentifier: Mono + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} - # Maui iOS scenario benchmarks (Mono Default) - Release - # Smoke coverage that the MAUI iOS app still builds and runs on Mono. - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - osx-x64-ios-arm64 - isPublic: false - jobParameters: - runKind: maui_scenarios_ios - projectFileName: maui_scenarios_ios.proj - channels: - - main - runtimeFlavor: mono - codeGenType: Default - buildConfig: Release - additionalJobIdentifier: Mono - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + # Maui iOS scenario benchmarks (CoreCLR Default) - Release + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - osx-x64-ios-arm64 + isPublic: false + jobParameters: + runKind: maui_scenarios_ios + projectFileName: maui_scenarios_ios.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: Default + buildConfig: Release + additionalJobIdentifier: CoreCLR + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} - # Maui iOS scenario benchmarks (CoreCLR Default) - Release - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - osx-x64-ios-arm64 - isPublic: false - jobParameters: - runKind: maui_scenarios_ios - projectFileName: maui_scenarios_ios.proj - channels: - - main - runtimeFlavor: coreclr - codeGenType: Default - buildConfig: Release - additionalJobIdentifier: CoreCLR - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + # Maui iOS scenario benchmarks (CoreCLR NativeAOT) - Release + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - osx-x64-ios-arm64 + isPublic: false + jobParameters: + runKind: maui_scenarios_ios + projectFileName: maui_scenarios_ios.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: NativeAOT + buildConfig: Release + additionalJobIdentifier: CoreCLR + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} - # Maui iOS scenario benchmarks (CoreCLR NativeAOT) - Release - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - osx-x64-ios-arm64 - isPublic: false - jobParameters: - runKind: maui_scenarios_ios - projectFileName: maui_scenarios_ios.proj - channels: - - main - runtimeFlavor: coreclr - codeGenType: NativeAOT - buildConfig: Release - additionalJobIdentifier: CoreCLR - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + # Maui Android scenario benchmarks (Mono - Default) - Debug + - ${{ if false }}: # disabled: iOS-only WIP test run + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-android-arm64-pixel + - win-x64-android-arm64-galaxy + isPublic: false + jobParameters: + runKind: maui_scenarios_android + projectFileName: maui_scenarios_android.proj + channels: + - main + runtimeFlavor: mono + codeGenType: Default + buildConfig: Debug + additionalJobIdentifier: Mono_Debug + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} # Maui Android scenario benchmarks (CoreCLR - Default) - Debug + - ${{ if false }}: # disabled: iOS-only WIP test run + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-android-arm64-pixel + - win-x64-android-arm64-galaxy + isPublic: false + jobParameters: + runKind: maui_scenarios_android + projectFileName: maui_scenarios_android.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: Default + buildConfig: Debug + additionalJobIdentifier: CoreCLR_Debug + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Maui Android inner loop benchmarks on devices (Mono Default) - Debug + - ${{ if false }}: # disabled: iOS-only WIP test run + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-android-arm64-pixel + - win-x64-android-arm64-galaxy + isPublic: false + jobParameters: + runKind: maui_scenarios_android_innerloop + projectFileName: maui_scenarios_android_innerloop.proj + channels: + - main + runtimeFlavor: mono + codeGenType: Default + buildConfig: Debug + additionalJobIdentifier: Mono_Debug_InnerLoop + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Maui Android inner loop benchmarks on devices (CoreCLR Default) - Debug + - ${{ if false }}: # disabled: iOS-only WIP test run + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-android-arm64-pixel + - win-x64-android-arm64-galaxy + isPublic: false + jobParameters: + runKind: maui_scenarios_android_innerloop + projectFileName: maui_scenarios_android_innerloop.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: Default + buildConfig: Debug + additionalJobIdentifier: CoreCLR_Debug_InnerLoop + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Maui Android inner loop benchmarks on emulator (Mono Default) - Debug + - ${{ if false }}: # disabled: iOS-only WIP test run + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - ubuntu-x64-android-emulator + isPublic: false + jobParameters: + runKind: maui_scenarios_android_innerloop + projectFileName: maui_scenarios_android_innerloop.proj + channels: + - main + runtimeFlavor: mono + codeGenType: Default + buildConfig: Debug + additionalJobIdentifier: Mono_Debug_InnerLoop_Emulator + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Maui Android inner loop benchmarks on emulator (CoreCLR Default) - Debug + - ${{ if false }}: # disabled: iOS-only WIP test run + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - ubuntu-x64-android-emulator + isPublic: false + jobParameters: + runKind: maui_scenarios_android_innerloop + projectFileName: maui_scenarios_android_innerloop.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: Default + buildConfig: Debug + additionalJobIdentifier: CoreCLR_Debug_InnerLoop_Emulator + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + - ${{ if false }}: # disabled for iOS inner-loop WIP; re-enable before merge + # Maui iOS scenario benchmarks (Mono - Default) - Debug + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - osx-x64-ios-arm64 + isPublic: false + jobParameters: + runKind: maui_scenarios_ios + projectFileName: maui_scenarios_ios.proj + channels: + - main + runtimeFlavor: mono + codeGenType: Default + buildConfig: Debug + additionalJobIdentifier: Mono_Debug + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Maui iOS scenario benchmarks (CoreCLR - Default) - Debug + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - osx-x64-ios-arm64 + isPublic: false + jobParameters: + runKind: maui_scenarios_ios + projectFileName: maui_scenarios_ios.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: Default + buildConfig: Debug + additionalJobIdentifier: CoreCLR_Debug + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Maui iOS Inner Loop (Mono) - Debug - Simulator - template: /eng/pipelines/templates/build-machine-matrix.yml parameters: jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml buildMachines: - - win-x64-android-arm64-pixel - - win-x64-android-arm64-galaxy + - osx-x64-ios-arm64-innerloop isPublic: false jobParameters: - runKind: maui_scenarios_android - projectFileName: maui_scenarios_android.proj + runKind: maui_scenarios_ios_innerloop + projectFileName: maui_scenarios_ios.proj channels: - main - runtimeFlavor: coreclr + runtimeFlavor: mono codeGenType: Default buildConfig: Debug - additionalJobIdentifier: CoreCLR_Debug + additionalJobIdentifier: Mono_InnerLoop_Simulator ${{ each parameter in parameters.jobParameters }}: ${{ parameter.key }}: ${{ parameter.value }} - # Maui Android inner loop benchmarks on devices (CoreCLR Default) - Debug + # Maui iOS Inner Loop (CoreCLR) - Debug - Simulator - template: /eng/pipelines/templates/build-machine-matrix.yml parameters: jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml buildMachines: - - win-x64-android-arm64-pixel - - win-x64-android-arm64-galaxy + - osx-x64-ios-arm64-innerloop isPublic: false jobParameters: - runKind: maui_scenarios_android_innerloop - projectFileName: maui_scenarios_android_innerloop.proj + runKind: maui_scenarios_ios_innerloop + projectFileName: maui_scenarios_ios.proj channels: - main runtimeFlavor: coreclr codeGenType: Default buildConfig: Debug - additionalJobIdentifier: CoreCLR_Debug_InnerLoop + additionalJobIdentifier: CoreCLR_InnerLoop_Simulator ${{ each parameter in parameters.jobParameters }}: ${{ parameter.key }}: ${{ parameter.value }} - # Maui Android inner loop benchmarks on emulator (CoreCLR Default) - Debug + # Maui iOS Inner Loop (Mono) - Debug - Device + # Device variants need the Helix Mac device queue's physical iPhone + code-signing infra + # (embedded.mobileprovision + sign tool). Those live on Mac.iPhone.17.Perf, but the queue + # config currently points at Mac.iPhone.13.Perf (the only queue with Xcode 26.5); device + # signing therefore needs provisioning on .13 (or Xcode 26.5 on .17). See build-machine-matrix.yml. - template: /eng/pipelines/templates/build-machine-matrix.yml parameters: jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml buildMachines: - - ubuntu-x64-android-emulator + - osx-x64-ios-arm64-innerloop isPublic: false jobParameters: - runKind: maui_scenarios_android_innerloop - projectFileName: maui_scenarios_android_innerloop.proj + runKind: maui_scenarios_ios_innerloop + projectFileName: maui_scenarios_ios.proj channels: - main - runtimeFlavor: coreclr + runtimeFlavor: mono codeGenType: Default buildConfig: Debug - additionalJobIdentifier: CoreCLR_Debug_InnerLoop_Emulator + additionalJobIdentifier: Mono_InnerLoop_Device + runEnvVars: + - iOSRid=ios-arm64 ${{ each parameter in parameters.jobParameters }}: ${{ parameter.key }}: ${{ parameter.value }} - # Maui iOS scenario benchmarks (CoreCLR - Default) - Debug + # Maui iOS Inner Loop (CoreCLR) - Debug - Device - template: /eng/pipelines/templates/build-machine-matrix.yml parameters: jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml buildMachines: - - osx-x64-ios-arm64 + - osx-x64-ios-arm64-innerloop isPublic: false jobParameters: - runKind: maui_scenarios_ios + runKind: maui_scenarios_ios_innerloop projectFileName: maui_scenarios_ios.proj channels: - main runtimeFlavor: coreclr codeGenType: Default buildConfig: Debug - additionalJobIdentifier: CoreCLR_Debug + additionalJobIdentifier: CoreCLR_InnerLoop_Device + runEnvVars: + - iOSRid=ios-arm64 ${{ each parameter in parameters.jobParameters }}: ${{ parameter.key }}: ${{ parameter.value }} @@ -626,30 +799,31 @@ jobs: ${{ parameter.key }}: ${{ parameter.value }} # NativeAOT scenario benchmarks - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - win-x64-viper - - ubuntu-x64-viper - - win-arm64-ampere - isPublic: false - jobParameters: - runKind: nativeaot_scenarios - projectFileName: nativeaot_scenarios.proj - channels: - - main - - 9.0 - - 8.0 - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + - ${{ if false }}: # disabled: iOS-only WIP test run + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-viper + - ubuntu-x64-viper + - win-arm64-ampere + isPublic: false + jobParameters: + runKind: nativeaot_scenarios + projectFileName: nativeaot_scenarios.proj + channels: + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} ################################################ # Scheduled Private jobs ################################################ # Scheduled runs will run all of the jobs on the PerfTigers, as well as the Arm64 job -- ${{ if parameters.runScheduledPrivateJobs }}: +- ${{ if false }}: # parameters.runScheduledPrivateJobs — disabled for iOS Inner Loop WIP # SDK scenario benchmarks - template: /eng/pipelines/templates/build-machine-matrix.yml diff --git a/eng/pipelines/templates/build-machine-matrix.yml b/eng/pipelines/templates/build-machine-matrix.yml index 232b94adf33..e7f5b7a312c 100644 --- a/eng/pipelines/templates/build-machine-matrix.yml +++ b/eng/pipelines/templates/build-machine-matrix.yml @@ -159,7 +159,7 @@ jobs: machinePool: AndroidEmulator ${{ insert }}: ${{ parameters.jobParameters }} -- ${{ if and(containsValue(parameters.buildMachines, 'osx-x64-ios-arm64'), not(eq(parameters.isPublic, true))) }}: # iPhone ARM64 17 only used in private builds currently +- ${{ if and(containsValue(parameters.buildMachines, 'osx-x64-ios-arm64'), not(eq(parameters.isPublic, true))) }}: # iPhone ARM64 13 (Mac.iPhone.13.Perf, updated to Xcode 26.5) - template: ${{ parameters.jobTemplate }} parameters: osGroup: osx @@ -167,8 +167,20 @@ jobs: osVersion: 26 pool: vmImage: 'macos-26' - queue: Mac.iPhone.17.Perf - machinePool: iPhone17 + queue: Mac.iPhone.13.Perf + machinePool: iPhone13 + ${{ insert }}: ${{ parameters.jobParameters }} + +- ${{ if and(containsValue(parameters.buildMachines, 'osx-x64-ios-arm64-innerloop'), not(eq(parameters.isPublic, true))) }}: # iOS Inner Loop: Mac.iPhone.13.Perf is the only queue with Xcode 26.5 (required by the net11.0_26.5 scenario). NOTE: it still lacks the PerfLab upload cert (LabCert1.pfx) and device code-signing infra (those live on Mac.iPhone.17.Perf, which only has Xcode 26.3) — pending infra provisioning so sim upload + device signing work here. + - template: ${{ parameters.jobTemplate }} + parameters: + osGroup: osx + archType: x64 + osVersion: 26 + pool: + vmImage: 'macos-26' + queue: Mac.iPhone.13.Perf + machinePool: iPhone13 ${{ insert }}: ${{ parameters.jobParameters }} - ${{ if and(containsValue(parameters.buildMachines, 'win-x64-viper'), not(eq(parameters.isPublic, true))) }}: # Windows x64 Viper only used in private builds diff --git a/eng/pipelines/templates/run-ios-innerloop-on-agent.sh b/eng/pipelines/templates/run-ios-innerloop-on-agent.sh new file mode 100755 index 00000000000..118a8026a8c --- /dev/null +++ b/eng/pipelines/templates/run-ios-innerloop-on-agent.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash +# +# Run the MAUI iOS inner-loop *simulator* perf scenario directly on the AzDO +# hosted macOS agent, instead of dispatching it to a Helix Mac device queue. +# +# The Helix Mac perf queues (Mac.iPhone.13/17.Perf) are EOL 2026-06-01 and need +# infra (Xcode) updates we can't get. The hosted `macos-26` image already ships +# Xcode + iOS simulators and is auto-updated, so for the simulator scenario we +# run setup_helix.py -> test.py -> post.py on the agent and emulate the small set +# of HELIX_* environment variables those scripts expect. +# +# Prerequisite: scripts/run_performance_job.py has already run on this agent and +# produced CorrelationStaging/payload, including: +# payload/dotnet - the .NET SDK +# payload/root/machine-setup.sh - PERFLAB_*/DOTNET_* exports +# payload/scripts, payload/shared - python modules (PYTHONPATH) +# payload/performance/src/scenarios/mauiiosinnerloop with the scaffolded `app/` +# and rollback_maui.json produced by `pre.py default` (PreparePayloadWorkItems). +# +# This driver intentionally mirrors the simulator HelixWorkItem in +# eng/performance/maui_scenarios_ios.proj (the _MSBuildArgs, scenario name, +# edit-src/edit-dest and test.py arguments). Keep the two in sync. + +set -uo pipefail + +payload_dir="" +runtime_flavor="" +framework="" +ios_rid="iossimulator-x64" +inner_loop_iterations="3" + +while [[ $# -gt 0 ]]; do + case "$1" in + --payload-dir) payload_dir="$2"; shift 2 ;; + --runtime-flavor) runtime_flavor="$2"; shift 2 ;; + --framework) framework="$2"; shift 2 ;; + --ios-rid) ios_rid="$2"; shift 2 ;; + --inner-loop-iterations) inner_loop_iterations="$2"; shift 2 ;; + *) echo "Unknown argument: $1" >&2; exit 1 ;; + esac +done + +if [[ -z "$payload_dir" || -z "$runtime_flavor" ]]; then + echo "Usage: $0 --payload-dir --runtime-flavor [--framework ] [--ios-rid ] [--inner-loop-iterations ]" >&2 + exit 1 +fi + +# Resolve to an absolute path so the rest of the script is location-independent. +payload_dir="$(cd "$payload_dir" && pwd)" +scenario_dir="$payload_dir/performance/src/scenarios/mauiiosinnerloop" + +if [[ ! -d "$scenario_dir" ]]; then + echo "Scenario directory not found: $scenario_dir" >&2 + echo "Did scripts/run_performance_job.py / PreparePayloadWorkItems run first?" >&2 + exit 1 +fi + +# Load PERFLAB_* / DOTNET_VERSION / PERFLAB_TARGET_FRAMEWORKS, etc. This is the +# same file the Helix work item sources via helix_pre_commands. +# Emulate the HELIX_* environment the scenario scripts (and the generated +# machine-setup.sh, which references $HELIX_CORRELATION_PAYLOAD) expect. These +# MUST be exported before sourcing machine-setup.sh. On Helix the correlation +# payload and the work item payload (the scenario dir) are unpacked separately; +# here we collapse the work item root onto the prepared scenario dir inside the +# payload, which is fine for a single local work item. +export HELIX_CORRELATION_PAYLOAD="$payload_dir" +export HELIX_WORKITEM_ROOT="$scenario_dir" +export HELIX_WORKITEM_UPLOAD_ROOT="$payload_dir/../uploadroot" +mkdir -p "$HELIX_WORKITEM_UPLOAD_ROOT" +HELIX_WORKITEM_UPLOAD_ROOT="$(cd "$HELIX_WORKITEM_UPLOAD_ROOT" && pwd)" +export HELIX_WORKITEM_UPLOAD_ROOT +export HELIX_WORKITEM_ID="MAUIiOSInnerLoop_Simulator_${runtime_flavor}_${BUILD_BUILDID:-local}" + +machine_setup="$payload_dir/root/machine-setup.sh" +if [[ -f "$machine_setup" ]]; then + echo "Sourcing $machine_setup" + # machine-setup.sh is generated for the Helix environment and may reference + # variables we don't set; relax nounset just for the source. + set +u + # shellcheck disable=SC1090 + source "$machine_setup" + set -u +else + echo "machine-setup.sh not found at $machine_setup" >&2 + exit 1 +fi + +# machine-setup.sh exports PERFLAB_TARGET_FRAMEWORKS (e.g. net11.0) but not +# PERFLAB_Framework, so prefer the explicitly passed framework and fall back to +# the sourced value. +framework="${framework:-}" +if [[ -z "$framework" ]]; then + framework="${PERFLAB_TARGET_FRAMEWORKS:-}" +fi +if [[ -z "$framework" ]]; then + echo "Could not determine target framework (no --framework and PERFLAB_TARGET_FRAMEWORKS unset)" >&2 + exit 1 +fi + +# Global Helix pre-commands (from Scenarios.Common.props) so `import shared.*` +# and `import performance.*` resolve. +export PYTHONPATH="$HELIX_CORRELATION_PAYLOAD/scripts:$HELIX_CORRELATION_PAYLOAD" + +# _MacEnvVars from maui_scenarios_ios.proj (machine-setup.sh already sets +# DOTNET_ROOT/PATH to the same values; re-export for parity with the proj). +export DOTNET_ROOT="$HELIX_CORRELATION_PAYLOAD/dotnet" +export DOTNET_CLI_TELEMETRY_OPTOUT=1 +export DOTNET_MULTILEVEL_LOOKUP=0 +export NUGET_PACKAGES="$HELIX_WORKITEM_ROOT/.packages" +export PATH="$DOTNET_ROOT:$PATH" +export IOS_RID="$ios_rid" + +# Install the Python packages the scenario needs at runtime (azure SDK for +# upload.py, cryptography, etc.). On Helix these are installed via the work +# item's helix_pre_commands; on the agent we must do it ourselves. runner.py +# imports `upload` unconditionally, so this is required even when not uploading. +# --break-system-packages matches the existing osx pip handling in +# scripts/run_performance_job.py. +requirements_file="$HELIX_CORRELATION_PAYLOAD/performance/requirements.txt" +if [[ -f "$requirements_file" ]]; then + echo "Installing Python requirements from $requirements_file" + python3 -m pip install --break-system-packages --user --disable-pip-version-check -q -r "$requirements_file" || { + echo "pip install of requirements failed" >&2 + exit 1 + } +else + echo "requirements.txt not found at $requirements_file" >&2 + exit 1 +fi + +# _MSBuildArgs from maui_scenarios_ios.proj (simulator path: iOSRid != ios-arm64). +if [[ "$runtime_flavor" == "mono" ]]; then + msbuild_args="/p:UseMonoRuntime=true" +elif [[ "$runtime_flavor" == "coreclr" ]]; then + msbuild_args="/p:UseMonoRuntime=false" +else + echo "Unsupported runtime flavor: $runtime_flavor" >&2 + exit 1 +fi +msbuild_args="$msbuild_args /p:RuntimeIdentifier=$ios_rid /p:MtouchLink=None" + +# Only upload to PerfLab from internal, non-PR runs (mirrors the gating in +# run-performance-job.yml). On the agent, upload.py authenticates via the +# AzureCLI@2 service-connection login (AzureCliCredential) that wraps this +# script; PERFLAB_UPLOAD_USE_AZURE_CLI selects that path over the Helix +# UAMI/cert flow. +scenario_args="" +if [[ "${SYSTEM_TEAMPROJECT:-}" != "public" && "${BUILD_REASON:-}" != "PullRequest" ]]; then + scenario_args="--upload-to-perflab-container" + export PERFLAB_UPLOAD_USE_AZURE_CLI=1 +fi + +echo "===== iOS inner loop (simulator) on-agent run =====" +echo " framework: $framework" +echo " runtime flavor: $runtime_flavor" +echo " iOS RID: $ios_rid" +echo " inner loop iters: $inner_loop_iterations" +echo " msbuild args: $msbuild_args" +echo " scenario args: ${scenario_args:-}" +echo " HELIX_WORKITEM_ROOT: $HELIX_WORKITEM_ROOT" +echo " HELIX_CORRELATION_PAYLOAD: $HELIX_CORRELATION_PAYLOAD" +echo "===================================================" + +cd "$HELIX_WORKITEM_ROOT" || exit 1 + +# PreCommands: setup (Xcode select, simulator boot, workload install, restore). +python3 setup_helix.py "$framework-ios" "$msbuild_args" || exit $? + +# Command: the actual inner-loop measurement. +rc=0 +python3 test.py iosinnerloop \ + --csproj-path app/MauiiOSInnerLoop.csproj \ + --edit-src "src/MainPage.xaml.cs;src/MainPage.xaml" \ + --edit-dest "app/Pages/MainPage.xaml.cs;app/Pages/MainPage.xaml" \ + --bundle-id com.companyname.mauiiosinnerloop \ + -f "$framework-ios" -c Debug \ + --msbuild-args "$msbuild_args" \ + --device-type simulator \ + --inner-loop-iterations "$inner_loop_iterations" \ + --scenario-name "Inner Loop Simulator - MAUI iOS Inner Loop" \ + $scenario_args || rc=$? + +# PostCommands: uninstall app, shut down build servers, clean. Best-effort. +python3 post.py || true + +exit $rc diff --git a/eng/pipelines/templates/run-performance-job.yml b/eng/pipelines/templates/run-performance-job.yml index 0045cdb5a03..5045c1f8fb0 100644 --- a/eng/pipelines/templates/run-performance-job.yml +++ b/eng/pipelines/templates/run-performance-job.yml @@ -36,6 +36,7 @@ parameters: targetCsproj: '' # optional -- Path to the csproj file to run benchmarks on runCategories: '' # optional -- Categories to run benchmarks on isScenario: false # optional -- Whether the job is a scenario job + runOnAgent: false # optional -- Run the scenario directly on the macOS agent instead of dispatching to a Helix Mac queue (iOS simulator inner loop) runEnvVars: [] # optional -- Environment variables to set for the benchmark runtimeFlavor: '' # optional -- Runtime flavor used for scenarios osVersion: '' # optional -- OS version to run on @@ -190,7 +191,7 @@ jobs: - '--is-scenario' - ${{ if ne(length(parameters.runEnvVars), 0) }}: - "--run-env-vars ${{ join(' ', parameters.runEnvVars)}}" - - ${{ if and(in(parameters.runKind, 'maui_scenarios_ios', 'maui_scenarios_android', 'maui_scenarios_android_innerloop'), ne(parameters.runtimeFlavor, '')) }}: + - ${{ if and(in(parameters.runKind, 'maui_scenarios_ios', 'maui_scenarios_android', 'maui_scenarios_ios_innerloop', 'maui_scenarios_android_innerloop'), ne(parameters.runtimeFlavor, '')) }}: - '--runtime-flavor ${{ parameters.runtimeFlavor }}' - ${{ if ne(parameters.osVersion, '') }}: - '--os-version ${{ parameters.osVersion }}' @@ -241,10 +242,31 @@ jobs: - '--cross-build' - ${{ if ne(parameters.additionalSetupParameters, '') }}: - '${{ parameters.additionalSetupParameters }}' - - template: /eng/pipelines/templates/send-to-helix-step.yml - parameters: - osGroup: ${{ parameters.osGroup }} - projectFile: $(_projectFile) + - ${{ if ne(parameters.runOnAgent, true) }}: + - template: /eng/pipelines/templates/send-to-helix-step.yml + parameters: + osGroup: ${{ parameters.osGroup }} + projectFile: $(_projectFile) + # DORMANT (kept until the Helix path is confirmed working): an alternative + # path that runs the iOS simulator inner loop scenario (setup_helix.py -> + # test.py -> post.py) directly on the macos-26 hosted agent instead of + # dispatching to a Helix Mac queue. Activated by runOnAgent: true on a job. + # No job sets it today, so this step is skipped and work goes to Helix. + # AzureCLI@2 + the '.NET Performance' service connection give upload.py's + # AzureCliCredential path access to pvscmdupload (blob OK; queue needs a + # Storage Queue Data role grant on the SP — that gap is why we run on Helix). + - ${{ if eq(parameters.runOnAgent, true) }}: + - task: AzureCLI@2 + displayName: 'Run iOS inner loop on agent (simulator)' + inputs: + azureSubscription: '.NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa)' + scriptType: bash + scriptLocation: scriptPath + scriptPath: $(performanceRepoDir)/eng/pipelines/templates/run-ios-innerloop-on-agent.sh + arguments: >- + --payload-dir "$(performanceRepoDir)/CorrelationStaging/payload" + --runtime-flavor ${{ parameters.runtimeFlavor }} + --framework "$(PERFLAB_Framework)" - ${{ if eq(parameters.osGroup, 'windows') }}: - task: PowerShell@2 displayName: Redact Logs diff --git a/scripts/run_performance_job.py b/scripts/run_performance_job.py index 5d93d189ab0..a2354399744 100644 --- a/scripts/run_performance_job.py +++ b/scripts/run_performance_job.py @@ -677,6 +677,15 @@ def get_run_configurations( if build_config is not None and build_config != DEFAULT_BUILD_CONFIG: configurations["BuildConfig"] = build_config + # .NET MAUI iOS inner loop (build+deploy) scenarios + if run_kind == "maui_scenarios_ios_innerloop": + if not runtime_flavor in ("mono", "coreclr"): + raise Exception("Runtime flavor must be specified for maui_scenarios_ios_innerloop") + configurations["CodegenType"] = str(codegen_type) + configurations["RuntimeType"] = str(runtime_flavor) + if build_config is not None and build_config != DEFAULT_BUILD_CONFIG: + configurations["BuildConfig"] = build_config + return configurations def get_work_item_command(os_group: str, target_csproj: str, architecture: str, perf_lab_framework: str, internal: bool, wasm: bool, bdn_artifacts_dir: str, wasm_coreclr: bool = False, only_sanity_check: bool = False): @@ -1268,6 +1277,21 @@ def publish_dotnet_app_to_payload(payload_dir_name: str, csproj_path: str, self_ os.environ["CodegenType"] = args.codegen_type or '' os.environ["BuildConfig"] = args.build_config or DEFAULT_BUILD_CONFIG + # Propagate run_env_vars to os.environ so they reach MSBuild + # evaluation as properties (e.g., iOSRid for device builds). + if args.run_env_vars: + for key, value in args.run_env_vars.items(): + os.environ[key] = value + + # Propagate RunKind so the PreparePayloadWorkItems msbuild call + # below evaluates _IsInnerLoop correctly and selects the inner-loop + # PreparePayloadWorkItem items (which run `pre.py default` and + # create rollback_maui.json in the payload dir). Without this, + # inner loop scenarios silently fall through to regular `pre.py + # publish` items and ship an incomplete payload to Helix. + if args.run_kind: + os.environ["RunKind"] = args.run_kind + # TODO: See if these commands are needed for linux as they were being called before but were failing. if args.os_group == "windows" or args.os_group == "osx": break_system_packages = ["--break-system-packages"] if args.os_group == "osx" else [] @@ -1297,7 +1321,7 @@ def publish_dotnet_app_to_payload(payload_dir_name: str, csproj_path: str, self_ verbose=True).run() # Search for additional binlogs generated by the maui scenarios prepare payload work items to copy to the artifacts log dir - if args.run_kind in ["maui_scenarios_android", "maui_scenarios_ios", "maui_scenarios_android_innerloop"]: + if args.run_kind in ["maui_scenarios_android", "maui_scenarios_ios", "maui_scenarios_ios_innerloop", "maui_scenarios_android_innerloop"]: for binlog_path in glob(os.path.join(payload_dir, "scenarios_out", "**", "*.binlog"), recursive=True): shutil.copy(binlog_path, ci_artifacts_log_dir) @@ -1428,10 +1452,21 @@ def get_work_item_command_for_artifact_dir(artifact_dir: str): only_sanity_check=args.only_sanity_check, ios_strip_symbols=args.ios_strip_symbols, ios_llvm_build=args.ios_llvm_build, + ios_rid=args.run_env_vars.get("iOSRid"), + run_kind=args.run_kind, fail_on_test_failure=fail_on_test_failure, scenario_arguments=scenario_arguments or None) if args.send_to_helix: + # Re-apply run_env_vars so they reach SendToHelix MSBuild evaluation. + # The env was snapshot/restored earlier (environ_copy), and while + # os.environ.update() preserves new keys, some shell wrappers may not + # inherit them reliably. Explicitly re-setting ensures properties like + # iOSRid (used in .proj ItemGroup conditions) are available. + if args.run_env_vars: + for key, value in args.run_env_vars.items(): + os.environ[key] = value + perf_send_to_helix(perf_send_to_helix_args) results_glob = os.path.join(helix_results_destination_dir, '**', '*perf-lab-report.json') diff --git a/scripts/send_to_helix.py b/scripts/send_to_helix.py index f3266623e3a..c7a883960ed 100644 --- a/scripts/send_to_helix.py +++ b/scripts/send_to_helix.py @@ -71,8 +71,10 @@ class PerfSendToHelixArgs: linking_type: Optional[str] = None python: Optional[str] = None affinity: Optional[str] = None + ios_rid: Optional[str] = None ios_strip_symbols: Optional[bool] = None ios_llvm_build: Optional[bool] = None + run_kind: Optional[str] = None scenario_arguments: Optional[list[str]] = None def set_environment_variables(self, save_to_pipeline: bool = True): @@ -111,8 +113,10 @@ def set_env_var(name: str, value: Union[str, bool, list[str], timedelta, int, No set_env_var("RuntimeFlavor", self.runtime_flavor) set_env_var("CodegenType", self.codegen_type) set_env_var("LinkingType", self.linking_type) + set_env_var("iOSRid", self.ios_rid) set_env_var("iOSStripSymbols", self.ios_strip_symbols) set_env_var("iOSLlvmBuild", self.ios_llvm_build) + set_env_var("RunKind", self.run_kind) set_env_var("TargetCsproj", self.target_csproj) set_env_var("WorkItemCommand", self.work_item_command, sep=" ") set_env_var("BaselineWorkItemCommand", self.baseline_work_item_command, sep=" ") @@ -142,5 +146,18 @@ def perf_send_to_helix(args: PerfSendToHelixArgs): binlog_dest = os.path.join(args.performance_repo_dir, "artifacts", "log", args.build_config, "SendToHelix.binlog") send_params = [args.project_file, "/restore", "/t:Test", f"/bl:{binlog_dest}"] + # Pass iOSRid explicitly as an MSBuild property so it reaches .proj + # evaluation reliably. Env var inheritance through msbuild.sh/tools.sh + # is unreliable for this property. + if args.ios_rid: + send_params.append(f"/p:iOSRid={args.ios_rid}") + + # Pass RunKind explicitly as an MSBuild property so the shared + # maui_scenarios_ios.proj can switch between regular and inner-loop + # ItemGroups based on $(RunKind). Env var inheritance through + # msbuild.sh/tools.sh is unreliable for this property. + if args.run_kind: + send_params.append(f"/p:RunKind={args.run_kind}") + run_msbuild_command(send_params, warn_as_error=False) diff --git a/scripts/upload.py b/scripts/upload.py index d76286a61b5..e02b142a5a7 100644 --- a/scripts/upload.py +++ b/scripts/upload.py @@ -41,6 +41,23 @@ def _try_managed_identity(managed_identity_client_id: Optional[str] = None): return None def get_credential(): + # 0. On an AzDO hosted agent (no Helix UAMI/cert) the run is wrapped in an + # AzureCLI@2 task that `az login`s as the '.NET Performance' service + # connection, which has write access to the pvscmdupload storage account. + # Use that AzureCliCredential directly for a Storage data-plane token. The + # Helix federated-assertion flow below does NOT work for this SP + # (AADSTS7002341 on api://AzureADTokenExchange). Opt-in via env var so Helix + # behavior is unchanged; fail closed (no cert fallback) when selected. + if os.environ.get("PERFLAB_UPLOAD_USE_AZURE_CLI", "").lower() in ("1", "true"): + from azure.identity import AzureCliCredential + getLogger().info("Attempting auth with Azure CLI credential (service connection).") + # The default AzureCliCredential subprocess timeout is 10s; on a loaded + # hosted agent cold-starting `az account get-access-token` can exceed + # that, so allow more time. + credential = AzureCliCredential(process_timeout=300) + credential.get_token("https://storage.azure.com/.default") + return credential + # 1. Try system-assigned managed identity getLogger().info("Attempting auth with system-assigned managed identity.") credential = _try_managed_identity() diff --git a/src/scenarios/mauiiosinnerloop/.gitignore b/src/scenarios/mauiiosinnerloop/.gitignore new file mode 100644 index 00000000000..cd91a583afb --- /dev/null +++ b/src/scenarios/mauiiosinnerloop/.gitignore @@ -0,0 +1,4 @@ +# Local measurement output produced by run-local.sh — never commit. +# (run-local.sh copies traces/*.binlog and *-versions.json into results// +# for local inspection. Production results go to the perf-lab via upload.py.) +results/ diff --git a/src/scenarios/mauiiosinnerloop/post.py b/src/scenarios/mauiiosinnerloop/post.py new file mode 100644 index 00000000000..c2d8dc0802a --- /dev/null +++ b/src/scenarios/mauiiosinnerloop/post.py @@ -0,0 +1,52 @@ +''' +post cleanup script +''' + +import os +import subprocess +import sys +import traceback +from performance.logger import setup_loggers, getLogger +from shared.ioshelper import iOSHelper +from shared.postcommands import clean_directories +from test import EXENAME + +setup_loggers(True) +logger = getLogger(__name__) + +try: + bundle_id = f'com.companyname.{EXENAME.lower()}' + ios_rid = os.environ.get('IOS_RID', 'iossimulator-arm64') + is_physical = (ios_rid == 'ios-arm64') + + helper = iOSHelper() + try: + if is_physical: + device_udid = iOSHelper.detect_connected_device() + if device_udid: + helper.setup_device(bundle_id, None, device_udid, is_physical=True) + helper.cleanup() + else: + logger.warning("No device UDID available — skipping uninstall") + else: + helper.setup_device(bundle_id, None, 'booted', is_physical=False) + helper.cleanup() + except Exception as e: + logger.warning("iOS uninstall failed (continuing): %s", e) + + workitem_root = os.environ.get('HELIX_WORKITEM_ROOT', '') + sim_udid_path = os.path.join(workitem_root, 'sim_udid.txt') if workitem_root else '' + if sim_udid_path and os.path.isfile(sim_udid_path): + try: + with open(sim_udid_path, 'r', encoding='utf-8') as f: + created_udid = f.read().strip() + if created_udid: + logger.info("Deleting per-workitem simulator: %s", created_udid) + iOSHelper.delete_simulator(created_udid) + except Exception as e: + logger.warning("Could not delete per-workitem simulator: %s", e) + + subprocess.run(['dotnet', 'build-server', 'shutdown'], check=False) + clean_directories() +except Exception as e: + logger.warning(f"Post cleanup encountered an error (best-effort, continuing): {e}\n{traceback.format_exc()}") diff --git a/src/scenarios/mauiiosinnerloop/pre.py b/src/scenarios/mauiiosinnerloop/pre.py new file mode 100644 index 00000000000..931bd41211b --- /dev/null +++ b/src/scenarios/mauiiosinnerloop/pre.py @@ -0,0 +1,547 @@ +''' +pre-command: Set up a MAUI iOS app for deploy measurement. +Creates the template (without restore) and prepares the modified file for incremental deploy. +NuGet packages are restored on the Helix machine, not shipped in the payload. +''' +import glob +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import urllib.request +import xml.etree.ElementTree as ET +import zipfile +from performance.common import get_repo_root_path +from performance.logger import setup_loggers, getLogger +from shared import const +from shared.mauisharedpython import install_latest_maui, MauiNuGetConfigContext +from shared.precommands import PreCommands +from test import EXENAME + +def install_maui_ios_workload(precommands: PreCommands): + ''' + Install the maui-ios workload (not the full 'maui' workload). + The full 'maui' workload includes Android/Windows components that aren't + needed for this scenario. Since this scenario only needs iOS, 'maui-ios' + is sufficient and much smaller. + + Uses the shared install_latest_maui() for rollback file creation and + workload install. Falls back to a manifest-patching workaround when + the iOS workload manifest references net10.0 cross-targeting packs + that don't exist on any NuGet feed (upstream coherency issue). + ''' + logger.info("########## Installing maui-ios workload ##########") + + if precommands.has_workload: + logger.info("Skipping maui-ios installation due to --has-workload=true") + return + + # Try the standard shared install path first. + try: + install_latest_maui( + precommands, + workloads=["microsoft.net.sdk.ios"], + workload_name='maui-ios', + ) + logger.info("########## Finished installing maui-ios workload ##########") + return + except subprocess.CalledProcessError as e: + logger.warning( + f"Standard workload install failed: {e}. " + "This typically happens when the iOS workload manifest (e.g. v26.4) " + "declares net10.0 cross-targeting packs that don't exist on any NuGet feed. " + "Falling back to manifest-patching workaround." + ) + + # ── Manifest-patching fallback ── + # The iOS workload manifest can reference net10.0 cross-targeting packs that + # haven't been published yet (upstream coherency issue). We work around this + # by downloading the manifest nupkg, patching out net10.0 entries, placing the + # patched manifest on disk, and installing with --skip-manifest-update. + logger.info("########## Starting manifest-patching fallback ##########") + + # Read package info from rollback_maui.json (created by install_latest_maui) + with open("rollback_maui.json", "r", encoding="utf-8") as f: + rollback_data = json.load(f) + ios_rollback = rollback_data.get("microsoft.net.sdk.ios", "") + if not ios_rollback or "/" not in ios_rollback: + raise Exception( + f"rollback_maui.json has no valid microsoft.net.sdk.ios entry: {rollback_data}" + ) + package_version, sdk_band = ios_rollback.split("/", 1) + # Reconstruct the manifest package ID from the SDK band + package_id = f"Microsoft.NET.Sdk.iOS.Manifest-{sdk_band}" + + # Look up the NuGet feed that matches the sdk_band from rollback_maui.json. + # install_latest_maui() may have fallen back to a different feed (e.g., dotnet10 + # instead of dotnet11) when creating the rollback. Using the wrong feed here + # causes HTTP 404 when downloading the manifest nupkg. + sdk_major = sdk_band.split('.')[0] # e.g., "10" from "10.0.100-preview.4" + nuget_config_path = os.path.join(get_repo_root_path(), "NuGet.config") + tree = ET.parse(nuget_config_path) + package_sources = tree.getroot().find(".//packageSources") + feed = None + for add_el in package_sources.findall("add"): + if add_el.get("key", "") == f"dotnet{sdk_major}": + feed = add_el.get("value") + break + if not feed: + raise Exception( + f"No NuGet feed found for dotnet{sdk_major} in {nuget_config_path}. " + f"sdk_band={sdk_band}, rollback entry={ios_rollback}" + ) + + # Step 1: Resolve PackageBaseAddress from the NuGet v3 service index + logger.info(f"Fetching NuGet v3 service index from {feed}") + with urllib.request.urlopen(feed, timeout=60) as resp: + service_index = json.loads(resp.read().decode('utf-8')) + + package_base_url = None + for resource in service_index.get('resources', []): + if 'PackageBaseAddress' in resource.get('@type', ''): + package_base_url = resource['@id'].rstrip('/') + break + if not package_base_url: + raise Exception( + f"Could not find PackageBaseAddress resource in NuGet v3 service index at {feed}" + ) + logger.info(f"Resolved PackageBaseAddress: {package_base_url}") + + # Step 2: Download the manifest nupkg (NuGet flat container uses lowercase IDs/versions) + nupkg_url = ( + f"{package_base_url}/{package_id.lower()}/{package_version.lower()}" + f"/{package_id.lower()}.{package_version.lower()}.nupkg" + ) + logger.info(f"Downloading manifest nupkg from {nupkg_url}") + + with tempfile.TemporaryDirectory() as tmpdir: + nupkg_path = os.path.join(tmpdir, 'manifest.nupkg') + # urlretrieve doesn't support timeout — use urlopen + manual write instead. + # Explicit timeout prevents Helix jobs from hanging indefinitely on network issues. + with urllib.request.urlopen(nupkg_url, timeout=120) as resp: + with open(nupkg_path, 'wb') as dl_file: + dl_file.write(resp.read()) + logger.info(f"Downloaded manifest nupkg to {nupkg_path}") + + # Step 3: Extract all files from the data/ directory inside the nupkg + extracted_files = {} + with zipfile.ZipFile(nupkg_path, 'r') as zf: + for entry in zf.namelist(): + if entry.startswith('data/') and not entry.endswith('/'): + filename = os.path.basename(entry) + extracted_files[filename] = zf.read(entry) + logger.info(f"Extracted from nupkg: {entry} ({len(extracted_files[filename])} bytes)") + + if 'WorkloadManifest.json' not in extracted_files: + raise Exception( + f"WorkloadManifest.json not found in data/ directory of {nupkg_url}. " + f"Found files: {list(extracted_files.keys())}" + ) + + # Step 4: Patch WorkloadManifest.json — remove entries containing "net10" + raw_text = extracted_files['WorkloadManifest.json'].decode('utf-8') + # The manifest uses trailing commas (invalid JSON). Strip them before parsing. + # Python's json.loads() (especially 3.14+) rejects trailing commas. + cleaned_text = re.sub(r',\s*([}\]])', r'\1', raw_text) + manifest = json.loads(cleaned_text) + + removed_packs = [] + removed_extends = [] + removed_top_packs = [] + + # Patch workloads.ios.packs — remove entries containing "net10" (case-insensitive). + # The packs field can be a list (of pack name strings) or a dict (pack names as keys), + # depending on the manifest version. + ios_workload = manifest.get('workloads', {}).get('ios', {}) + if 'packs' in ios_workload: + if isinstance(ios_workload['packs'], list): + original_items = list(ios_workload['packs']) + ios_workload['packs'] = [ + item for item in ios_workload['packs'] + if 'net10' not in item.lower() + ] + removed_packs = [item for item in original_items if 'net10' in item.lower()] + else: + original_keys = list(ios_workload['packs'].keys()) + for key in original_keys: + if 'net10' in key.lower(): + del ios_workload['packs'][key] + removed_packs.append(key) + + # Patch workloads.ios.extends — remove list items containing "net10" + if 'extends' in ios_workload: + original_extends = list(ios_workload['extends']) + ios_workload['extends'] = [ + item for item in ios_workload['extends'] + if 'net10' not in item.lower() + ] + removed_extends = [ + item for item in original_extends + if 'net10' in item.lower() + ] + + # Patch top-level packs — remove keys containing "net10". + # This should always be a dict, but guard against unexpected list format. + if 'packs' in manifest: + if isinstance(manifest['packs'], dict): + original_keys = list(manifest['packs'].keys()) + for key in original_keys: + if 'net10' in key.lower(): + del manifest['packs'][key] + removed_top_packs.append(key) + else: + logger.warning( + f"Top-level 'packs' is {type(manifest['packs']).__name__}, expected dict — " + "skipping top-level packs patching" + ) + + logger.info( + f"Patched WorkloadManifest.json — removed: " + f"workloads.ios.packs={removed_packs}, " + f"workloads.ios.extends={removed_extends}, " + f"top-level packs={removed_top_packs}" + ) + + if not removed_packs and not removed_extends and not removed_top_packs: + logger.warning( + "Manifest patching removed NO entries — the install failure may not be " + "caused by missing net10 packs. The subsequent install will likely fail " + "with the same error." + ) + + # json.dumps produces valid JSON (no trailing commas) — the SDK accepts both. + patched_json = json.dumps(manifest, indent=2) + extracted_files['WorkloadManifest.json'] = patched_json.encode('utf-8') + + # Step 5: Determine DOTNET_ROOT + dotnet_root = os.environ.get('DOTNET_ROOT') + if not dotnet_root: + dotnet_path = shutil.which('dotnet') + if not dotnet_path: + raise Exception("Cannot determine DOTNET_ROOT: not set and dotnet not found in PATH") + dotnet_root = os.path.dirname(os.path.realpath(dotnet_path)) + logger.info(f"DOTNET_ROOT: {dotnet_root}") + + # Step 6: Place patched manifest files on disk + # sdk_band comes from the manifest package ID (e.g., "11.0.100-preview.4") + # package_version is the NuGet version (e.g., "26.4.11427-net11-p4") + target_dir = os.path.join( + dotnet_root, 'sdk-manifests', sdk_band, + 'microsoft.net.sdk.ios', package_version + ) + os.makedirs(target_dir, exist_ok=True) + logger.info(f"Writing patched manifest files to {target_dir}") + + for filename, content in extracted_files.items(): + target_path = os.path.join(target_dir, filename) + with open(target_path, 'wb') as f: + f.write(content) + logger.info(f"Wrote {target_path} ({len(content)} bytes)") + + # Step 7: Install with --skip-manifest-update (manifest is already on disk) + logger.info("Installing maui-ios workload with --skip-manifest-update (patched manifest)") + precommands.install_workload('maui-ios', ['--skip-manifest-update']) + logger.info("########## Finished installing maui-ios workload (manifest-patched) ##########") + +def check_xcode_compatibility(framework: str): + ''' + Best-effort check that the active Xcode version matches the iOS SDK's + _RecommendedXcodeVersion. Logs a warning on mismatch — does not fail. + The caller (pipeline or run-local.sh) handles the actual Xcode selection. + ''' + try: + result = subprocess.run( + ['xcodebuild', '-version'], + capture_output=True, text=True, timeout=10 + ) + if result.returncode != 0: + logger.warning("Could not detect Xcode version (xcodebuild -version failed)") + return + + # Parse "Xcode 26.4" → "26.4" + xcode_line = result.stdout.strip().split('\n')[0] + xcode_match = re.search(r'Xcode\s+(\d+\.\d+)', xcode_line) + if not xcode_match: + logger.warning(f"Could not parse Xcode version from: {xcode_line}") + return + active_xcode = xcode_match.group(1) + + # Extract TFM prefix: "net11.0-ios" → "net11.0" + tfm_prefix = framework.split('-')[0] if '-' in framework else framework + + # Find the iOS SDK Versions.props file + dotnet_path = shutil.which('dotnet') + if not dotnet_path: + logger.warning("Could not find dotnet in PATH — skipping Xcode version check") + return + dotnet_dir = os.path.dirname(os.path.realpath(dotnet_path)) + packs_dir = os.path.join(dotnet_dir, 'packs') + + # Search for Microsoft.iOS.Sdk._*/*/targets/Microsoft.iOS.Sdk.Versions.props + search_pattern = os.path.join( + packs_dir, + f'Microsoft.iOS.Sdk.{tfm_prefix}_*', + '*', 'targets', 'Microsoft.iOS.Sdk.Versions.props' + ) + props_files = sorted(glob.glob(search_pattern)) + if not props_files: + logger.warning( + f"Could not find Microsoft.iOS.Sdk.Versions.props for {tfm_prefix} " + f"in {packs_dir} — skipping Xcode version check" + ) + return + + # Use the last (highest version) match + versions_props = props_files[-1] + logger.info(f"Found iOS SDK Versions.props: {versions_props}") + + # Parse _RecommendedXcodeVersion + with open(versions_props, 'r') as f: + content = f.read() + rec_match = re.search(r'<_RecommendedXcodeVersion>([^<]+)', content) + if not rec_match: + logger.warning(f"Could not find _RecommendedXcodeVersion in {versions_props}") + return + required_xcode = rec_match.group(1).strip() + + active_major_minor = '.'.join(active_xcode.split('.')[:2]) + required_major_minor = '.'.join(required_xcode.split('.')[:2]) + + if active_major_minor != required_major_minor: + logger.warning( + f"Xcode version MISMATCH: " + f"active Xcode is {active_xcode} but iOS SDK requires {required_xcode}. " + f"The build may fail with _ValidateXcodeVersion error. " + f"Set XCODE_PATH or DEVELOPER_DIR to a compatible Xcode installation." + ) + else: + logger.info( + f"Xcode version OK: active={active_xcode}, " + f"required={required_xcode} (major.minor match: {active_major_minor})" + ) + except Exception as e: + logger.warning(f"Xcode compatibility check failed (non-fatal): {e}") + +def strip_non_ios_tfms(csproj_path: str, framework: str): + ''' + Strip non-iOS TargetFrameworks from the generated .csproj. + The MAUI template (since .NET 10+) generates multiple conditional + elements for android, ios, maccatalyst, and windows. + We replace all of them with a single unconditional + containing only the iOS TFM we want to build. + Uses ]*> to match both unconditional and + Condition="..." variants of the element. + ''' + with open(csproj_path, 'r') as f: + content = f.read() + + logger.info(f"Stripping non-iOS TFMs from {csproj_path}, keeping: {framework}") + + # Remove all existing ... lines + # (both unconditional and conditional variants). + stripped = re.sub( + r'\s*]*>[^<]*', + '', + content + ) + + # Also handle singular if present + stripped = re.sub( + r'\s*]*>[^<]*', + '', + stripped + ) + + # Insert a single unconditional with the iOS TFM + # into the first + stripped = stripped.replace( + '', + f'\n {framework}', + 1 # only the first PropertyGroup + ) + + with open(csproj_path, 'w') as f: + f.write(stripped) + + logger.info(f"Stripped non-iOS TFMs. csproj now targets: {framework}") + +def inject_csproj_properties(csproj_path: str, properties: dict): + '''Inject MSBuild properties into the first PropertyGroup of a csproj.''' + with open(csproj_path, 'r') as f: + content = f.read() + + if '' not in content: + raise Exception(f"No found in {csproj_path}") + + for name, value in properties.items(): + if f'<{name}>' not in content: + content = content.replace( + '', + f' <{name}>{value}\n ', + 1 + ) + + with open(csproj_path, 'w') as f: + f.write(content) + logger.info(f"Injected properties into {csproj_path}: {list(properties.keys())}") + +def set_application_id(csproj_path: str, application_id: str): + '''Override the app's ApplicationId (bundle id). + + The MAUI template defaults to com.companyname., but device signing + uses the shared "NET Apple Development" provisioning profile whose wildcard + app-id is net.dot.* — the bundle id must fall under that prefix or the + device install is rejected. Simulator doesn't care, so we set it uniformly. + ''' + with open(csproj_path, 'r') as f: + content = f.read() + content, n = re.subn(r'[^<]*', + f'{application_id}', + content) + if n == 0: + content = content.replace( + '', + f' {application_id}\n ', + 1) + with open(csproj_path, 'w') as f: + f.write(content) + logger.info(f"Set ApplicationId to {application_id} in {csproj_path}") + +setup_loggers(True) +logger = getLogger(__name__) +logger.info("Starting pre-command for MAUI iOS deploy measurement") + +precommands = PreCommands() + +with MauiNuGetConfigContext(precommands.framework): + install_maui_ios_workload(precommands) + check_xcode_compatibility(precommands.framework) + precommands.print_dotnet_info() + + # Create template without restoring packages — packages will be restored + # on the Helix machine to avoid shipping ~1-2GB in the workitem payload. + precommands.new(template='maui', + output_dir=const.APPDIR, + bin_dir=const.BINDIR, + exename=EXENAME, + working_directory=sys.path[0], + no_restore=True) + + # Copy the merged NuGet.config into the app directory. This file contains + # MAUI NuGet feed URLs added by MauiNuGetConfigContext. The Helix machine + # needs these feeds during restore, and we must copy before the context + # manager restores the original NuGet.config. + repo_root = os.path.normpath(os.path.join(sys.path[0], '..', '..', '..')) + repo_nuget_config = os.path.join(repo_root, 'NuGet.config') + app_nuget_config = os.path.join(const.APPDIR, 'NuGet.config') + shutil.copy2(repo_nuget_config, app_nuget_config) + logger.info(f"Copied merged NuGet.config from {repo_nuget_config} to {app_nuget_config}") + + # Prepare the csproj: strip non-iOS TFMs and inject required properties + csproj_path = os.path.join(const.APPDIR, f'{EXENAME}.csproj') + strip_non_ios_tfms(csproj_path, precommands.framework) + inject_csproj_properties(csproj_path, { + # Preview SDKs may lack prune-package-data files, causing NETSDK1226. + 'AllowMissingPrunePackageData': 'true', + # Re-enable Roslyn compiler server (perf repo disables it globally + # for BenchmarkDotNet) to match real developer inner loop. + 'UseSharedCompilation': 'true', + }) + # Bundle id must fall under the device provisioning profile's net.dot.* wildcard. + set_application_id(csproj_path, 'net.dot.mauiiosinnerloop') + + # Create modified source files in src/ for the incremental deploy simulation. + # The runner toggles between original and modified versions each iteration, + # exercising both the C# compiler (Csc) and XAML compiler (XamlC) paths. + src_dir = os.path.join(sys.path[0], const.SRCDIR) + os.makedirs(src_dir, exist_ok=True) + + # --- Modified MainPage.xaml.cs: add a debug line in the constructor --- + # The template may place MainPage in either the root or Pages/ subdirectory. + # Normalize to ALWAYS use Pages/ so that the hardcoded --edit-dest paths in + # maui_scenarios_ios_innerloop.proj ("app/Pages/MainPage.xaml.cs") are valid. + pages_dir = os.path.join(const.APPDIR, 'Pages') + cs_candidates = [ + os.path.join(const.APPDIR, 'Pages', 'MainPage.xaml.cs'), + os.path.join(const.APPDIR, 'MainPage.xaml.cs'), + ] + cs_original = None + for candidate in cs_candidates: + if os.path.exists(candidate): + cs_original = candidate + break + if cs_original is None: + raise Exception( + "Could not find MainPage.xaml.cs in template — " + f"searched: {cs_candidates}" + ) + + # If MainPage files are at the root, move them into Pages/ so that the + # .proj's hardcoded edit-dest paths are always correct. + if os.path.dirname(os.path.abspath(cs_original)) != os.path.abspath(pages_dir): + os.makedirs(pages_dir, exist_ok=True) + for fname in ['MainPage.xaml.cs', 'MainPage.xaml']: + src_file = os.path.join(const.APPDIR, fname) + if os.path.exists(src_file): + shutil.move(src_file, os.path.join(pages_dir, fname)) + logger.info(f"Moved {src_file} → {os.path.join(pages_dir, fname)}") + cs_original = os.path.join(pages_dir, 'MainPage.xaml.cs') + + cs_modified = os.path.join(src_dir, 'MainPage.xaml.cs') + with open(cs_original, 'r') as f: + cs_content = f.read() + + cs_modified_content = cs_content.replace( + 'InitializeComponent();', + 'InitializeComponent();\n\t\tSystem.Diagnostics.Debug.WriteLine("incremental-touch");' + ) + if cs_modified_content == cs_content: + raise Exception( + "Could not find 'InitializeComponent();' in %s — template may have changed" % cs_original + ) + + with open(cs_modified, 'w') as f: + f.write(cs_modified_content) + logger.info(f"Modified MainPage.xaml.cs written to {cs_modified}") + + # --- Modified MainPage.xaml: change a label's text --- + # Look in the same directory where we found the .cs file + xaml_original = os.path.join(os.path.dirname(cs_original), 'MainPage.xaml') + if not os.path.exists(xaml_original): + raise Exception(f"Could not find MainPage.xaml at {xaml_original}") + + xaml_modified = os.path.join(src_dir, 'MainPage.xaml') + with open(xaml_original, 'r') as f: + xaml_content = f.read() + + # Use a flexible match — look for any Text="..." attribute on a Label + # to handle template variations. Prefer a known string first. + xaml_modified_content = xaml_content.replace( + 'Text="Hello, World!"', + 'Text="Hello, World! (updated)"' + ) + if xaml_modified_content == xaml_content: + # Fallback: try the .NET 10+ template's "Task Categories" text + xaml_modified_content = xaml_content.replace( + 'Text="Task Categories"', + 'Text="Task Categories (updated)"' + ) + if xaml_modified_content == xaml_content: + # Last resort: replace the first Text="..." attribute we find + xaml_modified_content = re.sub( + r'Text="([^"]*)"', + r'Text="\1 (updated)"', + xaml_content, + count=1 + ) + if xaml_modified_content == xaml_content: + raise Exception( + "Could not find any Text=\"...\" attribute in %s — template may have changed" % xaml_original + ) + + with open(xaml_modified, 'w') as f: + f.write(xaml_modified_content) + logger.info(f"Modified MainPage.xaml written to {xaml_modified}") diff --git a/src/scenarios/mauiiosinnerloop/select_xcode.py b/src/scenarios/mauiiosinnerloop/select_xcode.py new file mode 100644 index 00000000000..9379918ab21 --- /dev/null +++ b/src/scenarios/mauiiosinnerloop/select_xcode.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""select_xcode.py — Select the Xcode matching the iOS SDK version. + +Standalone (no repo imports). Prints selected path to stdout; logs to stderr. +""" +import argparse, glob, json, os, re, subprocess, sys +import xml.etree.ElementTree as ET + +_MIN_XCODE_MAJOR = 26 # Must match setup_helix.py + +def _log(msg): print(msg, file=sys.stderr) + +def _from_rollback(scenario_dir): + """Tier 1: Required Xcode major.minor from rollback_maui.json.""" + path = os.path.join(scenario_dir, "rollback_maui.json") + if not os.path.isfile(path): + return None + try: + with open(path, encoding="utf-8") as f: + ios = json.load(f).get("microsoft.net.sdk.ios", "") + # "26.2.11591-net11-p4/11.0.100-preview.3" → major=26, minor=2 + m = re.match(r"(\d+)\.(\d+)", ios.split("/")[0]) + return (int(m.group(1)), int(m.group(2))) if m else None + except (json.JSONDecodeError, OSError): + return None + +def _from_sdk_packs(dotnet_root): + """Tier 2: Required Xcode major.minor from SDK Versions.props.""" + if not dotnet_root: + return None + pattern = os.path.join(dotnet_root, "packs", "Microsoft.iOS.Sdk.*", + "*", "targets", "Microsoft.iOS.Sdk.Versions.props") + for props in sorted(glob.glob(pattern), reverse=True): + try: + for elem in ET.parse(props).iter("_RecommendedXcodeVersion"): + m = re.match(r"(\d+)\.(\d+)", elem.text or "") + if m: + return (int(m.group(1)), int(m.group(2))) + except ET.ParseError: + continue + return None + +def _parse_dir_version(name): + """'Xcode_26.2.app' or 'Xcode-26.2.0.app' → (26, 2) or (26, 2, 0).""" + stem = re.sub(r"^Xcode[_-]", "", name.replace(".app", "")) + try: + return tuple(int(p) for p in stem.split(".")) + except ValueError: + return None + +def _find_xcode(required): + """Find best Xcode in /Applications matching required (major, minor). + If required is None, picks the highest version >= _MIN_XCODE_MAJOR. + Also considers /Applications/Xcode.app via its embedded xcodebuild.""" + candidates = [] + for entry in sorted(os.listdir("/Applications")): + full = os.path.join("/Applications", entry) + if not (entry.endswith(".app") and os.path.isdir(full)): + continue + # Handles Xcode_26.2.app (Helix) and Xcode-26.2.0.app (Xcodes.app) + if re.match(r"Xcode[_-]\d+", entry): + ver = _parse_dir_version(entry) + elif entry == "Xcode.app": + try: + xb = os.path.join(full, "Contents/Developer/usr/bin/xcodebuild") + out = subprocess.run([xb, "-version"], capture_output=True, + text=True, timeout=10).stdout + m = re.search(r"Xcode\s+(\d+)\.(\d+)", out) + ver = (int(m.group(1)), int(m.group(2))) if m else None + except (OSError, subprocess.TimeoutExpired): + continue + else: + continue + if ver and len(ver) >= 2: + ok = (ver[0] == required[0] and ver[1] == required[1]) if required \ + else (ver[0] >= _MIN_XCODE_MAJOR) + if ok: + candidates.append((full, ver)) + candidates.sort(key=lambda x: x[1]) + return candidates[-1][0] if candidates else None + +if __name__ == "__main__": + p = argparse.ArgumentParser(description="Select Xcode matching iOS SDK") + p.add_argument("--scenario-dir", default=os.path.dirname(os.path.abspath(__file__))) + p.add_argument("--dotnet-root", default=os.environ.get("DOTNET_ROOT", "")) + args = p.parse_args() + + req = _from_rollback(args.scenario_dir) + if req: + _log(f"Required Xcode {req[0]}.{req[1]} (from rollback_maui.json)") + else: + req = _from_sdk_packs(args.dotnet_root) + if req: + _log(f"Required Xcode {req[0]}.{req[1]} (from SDK packs)") + else: + _log(f"No version constraint; selecting highest >= {_MIN_XCODE_MAJOR}") + selected = _find_xcode(req) + if not selected and req: + _log(f"No Xcode matching {req[0]}.{req[1]}; trying any >= {_MIN_XCODE_MAJOR}") + selected = _find_xcode(None) + if not selected: + _log("ERROR: No suitable Xcode found in /Applications/") + sys.exit(1) + _log(f"Selected: {selected}") + print(selected) diff --git a/src/scenarios/mauiiosinnerloop/setup_helix.py b/src/scenarios/mauiiosinnerloop/setup_helix.py new file mode 100644 index 00000000000..cab79e05c2a --- /dev/null +++ b/src/scenarios/mauiiosinnerloop/setup_helix.py @@ -0,0 +1,1288 @@ +#!/usr/bin/env python3 +"""setup_helix.py — Helix machine setup for MAUI iOS inner loop (macOS). + +Runs on the Helix machine BEFORE test.py. Bootstraps the macOS environment +for iOS builds: + 1. Configure DOTNET_ROOT and PATH from the correlation payload SDK. + 2. Select the Xcode version that matches the iOS SDK workload packs. + 3. Validate iOS simulator runtime availability. + 4. Boot the target iOS simulator device. + 5. (Device only) Locate signing artifacts; FAIL the work item if missing. + 6. Install the maui-ios workload. + 7. Restore NuGet packages for the app project. + 8. Disable Spotlight indexing on the workitem directory. + +Xcode selection strategy: The iOS SDK packs require a SPECIFIC Xcode version +(e.g. packs 26.2.x need Xcode 26.2). This script derives the required Xcode +major.minor from the ``rollback_maui.json`` file created by pre.py (shipped in +the workitem payload), then selects a matching ``/Applications/Xcode_*.app``. +If rollback_maui.json is absent or unparseable, it falls back to a coarse +``>= _MIN_XCODE_MAJOR`` check. This fails the work item early instead of +wasting 20+ minutes on workload install before hitting _ValidateXcodeVersion. + +Device path & infrastructure prerequisites +------------------------------------------ +For the physical-device variant (IOS_RID=ios-arm64) the build runs with +``EnableCodeSigning=false`` to keep MSBuild deterministic on Helix; the +post-build ``ioshelper.sign_app_for_device`` codesigns the .app by replicating +the Helix XHarness apple recipe: + + - download the provisioning profile from a blob (it is NOT installed on disk) + - unlock the ``signing-certs.keychain-db`` keychain (password in + ``~/.config/keychain``) and codesign with its Apple Development identity + +``verify_signing_prerequisites`` checks the keychain + password + a codesigning +identity before the (slow) build so a genuinely missing prerequisite fails the +work item early with a ``WORK ITEM FAILED — DEVICE INFRA UNAVAILABLE`` banner, +rather than after a full build. We deliberately do NOT mask the failure as a +"skip" / pass: a green build must mean the scenario actually ran. NOTE: earlier +versions searched the machine for a file literally named +``embedded.mobileprovision``, a ``sign`` tool, or a profile under +``~/Library/MobileDevice/Provisioning Profiles`` — all WRONG +(embedded.mobileprovision only exists inside already-signed bundles, no ``sign`` +tool exists on the perf Macs, and the profile is downloaded, not disk-installed), +which produced a false "DEVICE INFRA UNAVAILABLE". +""" + +import json +import os +import platform +import re +import subprocess +import sys +from datetime import datetime + +# --- Logging --- +# Follows the same logging pattern as the Android inner loop setup_helix.py: +# structured log file written to HELIX_WORKITEM_UPLOAD_ROOT for post-mortem +# debugging, with key messages also printed to stdout for Helix console output. +_logfile = None + + +def log(msg, tee=False): + """Write *msg* with a timestamp to the log file.""" + line = f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] {msg}" + if _logfile: + _logfile.write(line + "\n") + _logfile.flush() + if tee: + print(line, flush=True) + + +def log_raw(msg, tee=False): + """Write *msg* verbatim (no timestamp) to the log file.""" + if _logfile: + _logfile.write(msg + "\n") + _logfile.flush() + if tee: + print(msg, flush=True) + + +def run_cmd(args, check=True, **kwargs): + """Run a command, logging stdout/stderr. Returns CompletedProcess.""" + log(f"Running: {args}") + result = subprocess.run( + args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, + **kwargs, + ) + if result.stdout: + for line in result.stdout.splitlines(): + log_raw(line) + if check and result.returncode != 0: + raise subprocess.CalledProcessError(result.returncode, args, result.stdout) + return result + + +def _dump_log(): + """Print the full log file to stdout so it appears in Helix console output.""" + if not _logfile: + return + _logfile.flush() + try: + with open(_logfile.name, "r") as f: + print(f.read()) + except Exception: + pass + + +# --- Setup Steps --- + +def print_diagnostics(): + """Log environment variables useful for debugging Helix failures.""" + log_raw("=== DIAGNOSTICS ===", tee=True) + for var in ["DOTNET_ROOT", "PATH", "DEVELOPER_DIR", "HELIX_WORKITEM_ROOT", + "HELIX_CORRELATION_PAYLOAD", "TMPDIR", "USER", "HOME"]: + log_raw(f" {var}={os.environ.get(var, '')}") + log_raw(f" macOS version:", tee=True) + run_cmd(["sw_vers"], check=False) + log_raw(f" whoami / id:", tee=True) + run_cmd(["id"], check=False) + + +# CoreSimulator folders that may need to be owned by the current user before +# 'simctl boot' can succeed. On shared Helix machines, these folders may +# have accumulated state owned by a previous tenant (root, ado_agent, etc.), +# which causes 'simctl boot' to fail with NSCocoaErrorDomain code 513 +# ("You don't have permission to save the file ... in the folder +# CoreSimulator") — even for simulators we just created ourselves, because +# the boot writes log/state files into shared CoreSimulator folders we +# don't own. +_CORESIMULATOR_PATHS = [ + "/Library/Developer/CoreSimulator", + "/Library/Logs/CoreSimulator", + "~/Library/Developer/CoreSimulator", + "~/Library/Logs/CoreSimulator", + "~/Library/Caches/com.apple.CoreSimulator.SimulatorTrampoline", + "~/Library/Saved Application State/com.apple.CoreSimulator.CoreSimulatorService.savedState", +] + +# Subdirectory names underneath /Library/Developer/CoreSimulator that are +# Apple-managed read-only content (mounted runtime images, device-type +# bundles, signed system cryptexes). We have no permission to chown them and +# never need to — `simctl boot` only writes to Devices/ and Logs/. Without +# pruning these names, a recursive chown on /Library/Developer/CoreSimulator +# walks the iOS runtime volume (e.g. .../Volumes/iOS_23E254a/) and emits one +# "Operation not permitted" line per file — ~700k lines / 200+ MB of console +# log spam per Helix work item. +_CORESIMULATOR_PRUNE_NAMES = ("Volumes", "Profiles", "Cryptex", "Images") + + +def _sudo_chown_pruning(path, owner): + """``sudo chown -R owner path`` but prune Apple read-only subtrees. + + Equivalent to:: + + sudo find \\( -name Volumes -o -name Profiles -o + -name Cryptex -o -name Images \\) -prune \\ + -o -exec chown owner {} + + + Used in place of plain ``chown -R`` for the system-wide CoreSimulator + paths (see ``_CORESIMULATOR_PRUNE_NAMES`` for why). The plain ``chown -R`` + walks the iOS runtime image and emits hundreds of thousands of + "Operation not permitted" lines we then dutifully copy into the Helix + log; this variant skips those read-only mount points at the source. + """ + name_clause: list[str] = [] + for name in _CORESIMULATOR_PRUNE_NAMES: + if name_clause: + name_clause.append("-o") + name_clause.extend(["-name", name]) + cmd = ( + ["sudo", "find", path, "("] + name_clause + [")", "-prune", + "-o", "-exec", "chown", owner, "{}", "+"] + ) + return run_cmd(cmd, check=False) + + +def fix_coresimulator_permissions(): + """Take ownership of CoreSimulator folders so simctl boot can succeed. + + Helix machines are shared; CoreSimulator folders may have accumulated + state owned by prior tenants. ``simctl boot`` writes log/state files + into several Library folders, and even a freshly-created device fails + to boot if any one of them is not writable by the current user. This + function chowns the relevant folders to the current user so that boot, + spawn, and shutdown all work. + + For the system-wide ``/Library/Developer/CoreSimulator`` tree we use + ``find ... -prune`` rather than a plain ``chown -R`` to avoid recursing + into Apple's read-only runtime/profile/cryptex mount points (see + ``_CORESIMULATOR_PRUNE_NAMES``). For per-user paths under ``~/Library`` + we use plain ``chown -R`` because they don't contain those mounts. + + It is best-effort: paths that don't exist are skipped, and chown + failures are logged as warnings so the rest of setup still runs (the + boot itself will fail loudly with diagnostics if permissions are still + wrong). + """ + log_raw("=== COREsimulator PERMISSIONS ===", tee=True) + user = os.environ.get("USER") or "" + if not user: + # Fallback: query the OS + result = run_cmd(["whoami"], check=False) + user = (result.stdout or "").strip() + if not user: + log("WARNING: Cannot determine current user; skipping CoreSimulator chown", + tee=True) + return + owner = f"{user}:staff" + + for raw in _CORESIMULATOR_PATHS: + path = os.path.expanduser(raw) + if not os.path.exists(path): + log(f" skip (not present): {path}") + continue + # Only the system-wide /Library/Developer/CoreSimulator tree contains + # the Apple-mounted read-only volumes that explode chown -R output. + if path == "/Library/Developer/CoreSimulator": + log(f" sudo find {path} (prune {_CORESIMULATOR_PRUNE_NAMES}) " + f"-exec chown {owner}", tee=True) + result = _sudo_chown_pruning(path, owner) + else: + log(f" sudo chown -R {owner} {path}", tee=True) + result = run_cmd(["sudo", "chown", "-R", owner, path], check=False) + if result.returncode != 0: + log(f"WARNING: chown on {path} failed (exit {result.returncode}). " + f"simctl boot may still hit permission errors.", tee=True) + # Show ownership for diagnostics (top-level only — ls -la is non-recursive) + run_cmd(["ls", "-la", path], check=False) + + +def setup_dotnet(correlation_payload): + """Set DOTNET_ROOT and PATH to point at the SDK in the correlation payload. + + Returns the path to the dotnet executable. + """ + dotnet_root = os.path.join(correlation_payload, "dotnet") + dotnet_exe = os.path.join(dotnet_root, "dotnet") + + os.environ["DOTNET_ROOT"] = dotnet_root + os.environ["PATH"] = dotnet_root + ":" + os.environ.get("PATH", "") + + log(f"DOTNET_ROOT={dotnet_root}", tee=True) + run_cmd([dotnet_exe, "--version"], check=False) + return dotnet_exe + + +# Coarse pre-filter: packs are not yet installed at this point (workload +# install happens at step 5), so we can't read _RecommendedXcodeVersion from +# the pack's Versions.props. This >= 26 check catches clearly incompatible +# machines early. The SDK's _ValidateXcodeVersion target performs the exact +# version check at build time. +_MIN_XCODE_MAJOR = 26 + + +def _parse_xcode_version(output): + """Parse the major.minor version tuple from ``xcodebuild -version`` output. + + Expects a line like "Xcode 26.2" or "Xcode 15.0.1". + Returns (major, minor) as ints, or None if parsing fails. + """ + m = re.search(r"Xcode\s+(\d+)\.(\d+)", output or "") + if m: + return int(m.group(1)), int(m.group(2)) + return None + + +def _get_required_xcode_version(workitem_root): + """Derive the required Xcode major.minor from rollback_maui.json. + + The rollback file is created by pre.py and shipped in the workitem payload. + It contains e.g.:: + + {"microsoft.net.sdk.ios": "26.2.11591-net11-p4/11.0.100-preview.3"} + + The version prefix ``26.2`` IS the required Xcode version (major.minor). + Returns ``(major, minor)`` as ints, or ``None`` if the file is absent or + unparseable. + """ + rollback_path = os.path.join(workitem_root, "rollback_maui.json") + if not os.path.isfile(rollback_path): + log("rollback_maui.json not found — cannot derive required Xcode version") + return None + + try: + with open(rollback_path, "r", encoding="utf-8") as f: + data = json.load(f) + except (json.JSONDecodeError, OSError) as e: + log(f"WARNING: Failed to read rollback_maui.json: {e}") + return None + + ios_value = data.get("microsoft.net.sdk.ios") + if not ios_value: + log("WARNING: rollback_maui.json has no 'microsoft.net.sdk.ios' key") + return None + + # Value format: "26.2.11591-net11-p4/11.0.100-preview.3" + # Extract version prefix before the "/" (band), then parse major.minor + version_part = ios_value.split("/")[0] # "26.2.11591-net11-p4" + m = re.match(r"(\d+)\.(\d+)", version_part) + if not m: + log(f"WARNING: Could not parse major.minor from iOS SDK version '{version_part}'") + return None + + major, minor = int(m.group(1)), int(m.group(2)) + log(f"Required Xcode version from rollback_maui.json: {major}.{minor} " + f"(from '{ios_value}')", tee=True) + return (major, minor) + + +def _parse_xcode_dir_version(dirname): + """Parse version components from an Xcode directory name. + + ``"Xcode_26.2.app"`` → ``(26, 2)``, ``"Xcode_26.2.1.app"`` → ``(26, 2, 1)``. + Returns a tuple of ints, or ``None`` if parsing fails. + """ + # Strip "Xcode_" prefix and ".app" suffix, then split on "." + stem = dirname.replace("Xcode_", "").replace(".app", "") + parts = stem.split(".") + try: + return tuple(int(p) for p in parts) + except ValueError: + return None + + +def select_xcode(workitem_root): + """Select the Xcode version that matches the iOS SDK workload packs. + + Derives the required Xcode major.minor from rollback_maui.json (created by + pre.py). If the system-default Xcode already matches, no switching is + needed. Otherwise, searches /Applications/Xcode_*.app for a matching + version and activates it via ``sudo xcode-select -s``. Falls back to the + coarse ``>= _MIN_XCODE_MAJOR`` check when rollback_maui.json is absent. + """ + log_raw("=== XCODE SELECTION ===", tee=True) + + required = _get_required_xcode_version(workitem_root) + + # Log the current default before any changes + run_cmd(["xcode-select", "-p"], check=False) + result = run_cmd(["xcodebuild", "-version"], check=False) + + current = _parse_xcode_version(result.stdout) + if current is None: + log("ERROR: Could not parse Xcode version from xcodebuild output. " + "Ensure Xcode is installed.", tee=True) + _dump_log() + sys.exit(1) + + cur_major, cur_minor = current + log(f"Default Xcode version: {cur_major}.{cur_minor}", tee=True) + + # --- Precise matching mode (rollback file provides required version) --- + if required is not None: + req_major, req_minor = required + + if cur_major == req_major and cur_minor == req_minor: + log(f"Default Xcode {cur_major}.{cur_minor} matches required " + f"{req_major}.{req_minor} — no switching needed.", tee=True) + return + + # Default doesn't match — search for a matching Xcode_*.app + log(f"Default Xcode {cur_major}.{cur_minor} does not match required " + f"{req_major}.{req_minor}. Searching /Applications/Xcode_*.app...", + tee=True) + + matching_apps = [] + for entry in os.listdir("/Applications"): + if not (entry.startswith("Xcode_") and entry.endswith(".app")): + continue + if not os.path.isdir(os.path.join("/Applications", entry)): + continue + ver = _parse_xcode_dir_version(entry) + if ver and len(ver) >= 2 and ver[0] == req_major and ver[1] == req_minor: + matching_apps.append((entry, ver)) + + if not matching_apps: + # List all available Xcode installations for diagnostics + all_xcodes = sorted( + e for e in os.listdir("/Applications") + if e.startswith("Xcode") and e.endswith(".app") + ) + log(f"ERROR: No Xcode matching {req_major}.{req_minor} found. " + f"Available: {all_xcodes}", tee=True) + log(f"The iOS SDK packs require Xcode {req_major}.{req_minor}. " + "Install the matching Xcode or update the workload pin.", + tee=True) + _dump_log() + sys.exit(1) + + # If multiple match (e.g. Xcode_26.2.app and Xcode_26.2.1.app), + # pick the highest patch version + matching_apps.sort(key=lambda x: x[1]) + selected_name, selected_ver = matching_apps[-1] + selected = os.path.join("/Applications", selected_name) + + log(f"Found matching Xcode installations: " + f"{[name for name, _ in matching_apps]}", tee=True) + log(f"Selecting: {selected}", tee=True) + run_cmd(["sudo", "xcode-select", "-s", selected], check=False) + + # Log the new state after switching + run_cmd(["xcode-select", "-p"], check=False) + result = run_cmd(["xcodebuild", "-version"], check=False) + + version = _parse_xcode_version(result.stdout) + if version: + log(f"Xcode version after switching: {version[0]}.{version[1]}", + tee=True) + + # Validate the switch actually produced the required version + if version is None or version[0] != req_major or version[1] != req_minor: + effective = f"{version[0]}.{version[1]}" if version else "unknown" + log(f"ERROR: Xcode switch failed — active version is {effective} " + f"but {req_major}.{req_minor} is required. " + f"sudo xcode-select -s may have failed silently.", tee=True) + _dump_log() + sys.exit(1) + return + + # --- Fallback mode (no rollback file — coarse >= check) --- + log("WARNING: No rollback_maui.json — falling back to coarse " + f">= {_MIN_XCODE_MAJOR}.0 check", tee=True) + + if cur_major >= _MIN_XCODE_MAJOR: + log(f"Default Xcode {cur_major}.{cur_minor} meets minimum " + f"({_MIN_XCODE_MAJOR}.0) — no switching needed.", tee=True) + return + + # Default Xcode is too old — search for a newer Xcode_*.app + log(f"Default Xcode {cur_major}.{cur_minor} is below minimum " + f"({_MIN_XCODE_MAJOR}.0). " + "Searching for a newer /Applications/Xcode_*.app...", tee=True) + + xcode_apps = sorted( + ( + entry + for entry in os.listdir("/Applications") + if entry.startswith("Xcode_") and entry.endswith(".app") + and os.path.isdir(os.path.join("/Applications", entry)) + ), + key=lambda name: [ + int(x) for x in re.findall(r"\d+", name.replace(".app", "")) + ], + ) + + version = current + if xcode_apps: + selected = os.path.join("/Applications", xcode_apps[-1]) + log(f"Found Xcode installations: {xcode_apps}", tee=True) + log(f"Selecting highest version: {selected}", tee=True) + run_cmd(["sudo", "xcode-select", "-s", selected], check=False) + + # Log the new state after switching + run_cmd(["xcode-select", "-p"], check=False) + result = run_cmd(["xcodebuild", "-version"], check=False) + + version = _parse_xcode_version(result.stdout) + if version: + log(f"Xcode version after switching: {version[0]}.{version[1]}", + tee=True) + else: + log("WARNING: No /Applications/Xcode_*.app found.", tee=True) + + # Final validation — fail fast if still below minimum + if version is None or version[0] < _MIN_XCODE_MAJOR: + effective = f"{version[0]}.{version[1]}" if version else "unknown" + log(f"ERROR: Xcode {effective} is still below the minimum required " + f"version ({_MIN_XCODE_MAJOR}.0). The iOS SDK packs require " + f"Xcode >= {_MIN_XCODE_MAJOR}.0. Failing early to avoid wasting " + "time on workload install.", tee=True) + _dump_log() + sys.exit(1) + + +def validate_simulator_runtimes(): + """Check that iOS simulator runtimes are available on this machine.""" + log_raw("=== SIMULATOR RUNTIME VALIDATION ===", tee=True) + result = run_cmd(["xcrun", "simctl", "list", "runtimes"], check=False) + if result.returncode != 0: + log("WARNING: 'xcrun simctl list runtimes' failed. " + "Simulator may not work.", tee=True) + return + + # Check that at least one iOS runtime is listed + ios_runtimes = [line for line in (result.stdout or "").splitlines() + if "iOS" in line] + if ios_runtimes: + log(f"Found {len(ios_runtimes)} iOS runtime(s):", tee=True) + for rt in ios_runtimes: + log(f" {rt.strip()}") + else: + log("WARNING: No iOS simulator runtimes found. " + "Simulator-based testing will fail.", tee=True) + + +# Preferred iPhone device types in priority order. Used to pick a device type +# when creating our own simulator. We prefer plain models over Pro/Pro Max for +# predictable performance characteristics, and skip mini/SE variants because +# their availability across Xcode versions is inconsistent. The list is wide +# enough to survive Xcode/runtime upgrades on Helix machines. +_PREFERRED_IPHONE_MODELS = [ + "iPhone 17", "iPhone 17 Pro", "iPhone 17 Pro Max", "iPhone Air", + "iPhone 16", "iPhone 16 Pro", "iPhone 16 Pro Max", "iPhone 16 Plus", + "iPhone 15", "iPhone 15 Pro", "iPhone 15 Pro Max", "iPhone 15 Plus", + "iPhone 14", "iPhone 14 Pro", "iPhone 14 Pro Max", "iPhone 14 Plus", + "iPhone 13", "iPhone 13 Pro", "iPhone 13 Pro Max", + "iPhone 12", "iPhone 12 Pro", "iPhone 12 Pro Max", + "iPhone 11", "iPhone 11 Pro", "iPhone 11 Pro Max", +] + + +def _find_ios_runtime(): + """Return the (identifier, version) of the highest-version available iOS runtime. + + Returns ``(None, None)`` if no iOS runtime is available. + """ + result = run_cmd(["xcrun", "simctl", "list", "runtimes", "-j"], check=False) + if result.returncode != 0 or not result.stdout: + return (None, None) + + try: + data = json.loads(result.stdout) + except json.JSONDecodeError as e: + log(f"WARNING: Failed to parse runtimes JSON: {e}") + return (None, None) + + available = [] + for rt in data.get("runtimes", []): + if not rt.get("isAvailable"): + continue + name = rt.get("name", "") + if "iOS" not in name: + continue + identifier = rt.get("identifier", "") + version = rt.get("version", "0") + available.append((identifier, version, name)) + + if not available: + return (None, None) + + # Sort by version (descending) so the latest iOS runtime wins. version + # strings like "26.4.1" sort correctly with packaging-style key, but we + # only need a stable preference for the latest, so simple tuple sort works. + def _version_key(item): + try: + return tuple(int(p) for p in item[1].split(".") if p.isdigit()) + except Exception: + return (0,) + + available.sort(key=_version_key, reverse=True) + rt_id, rt_ver, rt_name = available[0] + log(f"Selected iOS runtime: {rt_name} (id={rt_id}, version={rt_ver})", tee=True) + return (rt_id, rt_ver) + + +def _find_iphone_device_type(): + """Return the identifier of a usable iPhone simulator device type. + + Walks ``_PREFERRED_IPHONE_MODELS`` in order and returns the identifier of + the first model present in ``simctl list devicetypes -j``. Returns ``None`` + if none of the preferred models are available. + """ + result = run_cmd(["xcrun", "simctl", "list", "devicetypes", "-j"], check=False) + if result.returncode != 0 or not result.stdout: + return None + + try: + data = json.loads(result.stdout) + except json.JSONDecodeError as e: + log(f"WARNING: Failed to parse devicetypes JSON: {e}") + return None + + by_name = {dt.get("name", ""): dt.get("identifier", "") + for dt in data.get("devicetypes", [])} + for model in _PREFERRED_IPHONE_MODELS: + if model in by_name and by_name[model]: + log(f"Selected iPhone device type: {model} (id={by_name[model]})", tee=True) + return by_name[model] + + log(f"WARNING: None of the preferred iPhone models are installed. " + f"Available iPhones: {[n for n in by_name if n.startswith('iPhone')]}", + tee=True) + return None + + +def _unique_simulator_name(): + """Build a per-workitem unique simulator name to avoid collisions. + + Each Helix work item gets its own name so concurrent work items on the + same machine never share or delete each other's simulators. + """ + workitem_id = (os.environ.get("HELIX_WORKITEM_ID") or + os.environ.get("HELIX_WORKITEM_FRIENDLYNAME") or + "") + if workitem_id: + # Strip filesystem-unfriendly chars from friendly names like + # "Inner Loop Simulator - MAUI iOS Inner Loop" + suffix = re.sub(r"[^A-Za-z0-9_-]+", "-", workitem_id).strip("-")[:48] + else: + # Fallback: epoch + PID. Still unique within a single machine session. + suffix = f"{int(datetime.now().timestamp())}-{os.getpid()}" + return f"PerfTest-iPhone-{suffix}" + + +def _write_sim_udid(workitem_root, udid): + """Persist the UDID of the simulator we created so test.py / post.py can + read it. Without this, ioshelper would fall back to "first booted device" + which is unsafe if any other simulator happens to be booted. + """ + path = os.path.join(workitem_root, "sim_udid.txt") + try: + with open(path, "w", encoding="utf-8") as f: + f.write(udid + "\n") + log(f"Wrote simulator UDID to {path}", tee=True) + except OSError as e: + log(f"WARNING: Could not write {path}: {e}", tee=True) + + +def _simulator_preflight(udid): + """Verify the booted simulator is actually usable for spawning processes. + + The CoreSimulator daemon can list a device as "Booted" while still + refusing to spawn agents (for example when the device's underlying data + container is owned by a different user — the failure mode that breaks + actool's AssetCatalogSimulatorAgent during ``dotnet build``). + Running ``simctl spawn /usr/bin/true`` proves end-to-end that we + can launch processes inside the simulator under the current user. + Raises ``SystemExit(1)`` if the spawn fails. + """ + log(f"Preflight: spawning /usr/bin/true inside simulator {udid}", tee=True) + result = run_cmd( + ["xcrun", "simctl", "spawn", udid, "/usr/bin/true"], + check=False, + ) + if result.returncode != 0: + log(f"ERROR: simctl spawn preflight failed (exit {result.returncode}). " + f"CoreSimulator cannot launch processes in this device. " + f"actool / mlaunch will also fail.", tee=True) + _dump_log() + sys.exit(1) + log("Preflight OK — simulator can spawn processes.", tee=True) + + +def _sweep_leaked_perftest_simulators(): + """Shutdown and delete any leaked ``PerfTest-iPhone-*`` simulators from + previous workitems on this Helix machine. + + Each booted simulator forks ~150–200 child processes (launchd_sim plus + a long tail of system daemons). A handful of leaked simulators is enough + to push the per-user process count past the macOS ``maxUserProcs`` + rlimit (typically 1333), at which point ``simctl boot`` for THIS work + item fails with NSPOSIXErrorDomain code 67 / "Unable to boot device + due to insufficient system resources". Leaks happen when a previous + workitem's post.py crashed before reaching delete_simulator(), or when + the workitem was killed mid-run by a Helix timeout. + + We're conservative: we only touch devices whose name starts with the + well-known ``PerfTest-iPhone-`` prefix used by ``_unique_simulator_name``, + so we never disturb shared queue infrastructure or non-perf workitems. + Best-effort — failures are logged but don't abort setup. + """ + log_raw("=== LEAKED PERFTEST SIMULATOR SWEEP ===", tee=True) + try: + listing = run_cmd(["xcrun", "simctl", "list", "devices", "-j"], + check=False) + except Exception as e: + log(f"WARNING: could not list simulators for sweep: {e}", tee=True) + return + if listing.returncode != 0 or not listing.stdout: + log(f"WARNING: simctl list devices -j failed (exit {listing.returncode}); " + "skipping sweep.", tee=True) + return + try: + devices_by_runtime = json.loads(listing.stdout).get("devices", {}) + except (json.JSONDecodeError, AttributeError) as e: + log(f"WARNING: could not parse simctl device JSON for sweep: {e}", + tee=True) + return + + leaked = [] + for runtime_devices in devices_by_runtime.values(): + for dev in runtime_devices or []: + name = (dev.get("name") or "").strip() + udid = (dev.get("udid") or "").strip() + if name.startswith("PerfTest-iPhone-") and udid: + leaked.append((name, udid, dev.get("state", "Unknown"))) + + if not leaked: + log("No leaked PerfTest-iPhone-* simulators found.", tee=True) + return + + log(f"Found {len(leaked)} leaked PerfTest-iPhone-* simulator(s); " + "shutting down and deleting...", tee=True) + for name, udid, state in leaked: + log(f" Cleaning {name} ({udid}, state={state})", tee=True) + # shutdown is best-effort and idempotent on already-shutdown devices + run_cmd(["xcrun", "simctl", "shutdown", udid], check=False) + run_cmd(["xcrun", "simctl", "delete", udid], check=False) + + +def create_and_boot_simulator(workitem_root): + """Create a fresh iPhone simulator under the current user's CoreSimulator + namespace and boot it. + + Why a fresh device every run: + Helix machines accumulate simulator devices across runs. When those + pre-existing devices are owned by a different user (root, or a previous + tenant), ``simctl boot`` on them fails with NSCocoaErrorDomain code 513 + ("You don't have permission to save the file ... in the folder + CoreSimulator"). Creating our own device with ``simctl create`` writes + to ``~/Library/Developer/CoreSimulator/Devices/`` of THIS user, so we + always have a writable device to boot. + + Why both simulator AND device jobs need this: + ``dotnet build`` for ios-arm64 invokes ``actool`` which spawns + ``AssetCatalogSimulatorAgent`` via CoreSimulator to compile the asset + catalog. Without a writable, booted simulator under our user, that + spawn fails and the build aborts. + + Returns the UDID of the booted device. Persists it to + ``/sim_udid.txt`` for test.py / post.py to read. + Exits with code 1 on any unrecoverable failure. + """ + log_raw("=== SIMULATOR CREATE & BOOT ===", tee=True) + + # Defensive cleanup: kill any leaked PerfTest-iPhone-* simulators from + # previous workitems before we try to boot our own. Without this, on a + # machine where prior post.py runs crashed or were killed by timeout, + # the per-user process count stays pinned just below maxUserProcs and + # this workitem's simctl boot fails with exit 67. Only touches devices + # we own (PerfTest-iPhone-* naming) so other workitems are unaffected. + _sweep_leaked_perftest_simulators() + + runtime_id, _ = _find_ios_runtime() + if not runtime_id: + log("ERROR: No available iOS simulator runtime found.", tee=True) + run_cmd(["xcrun", "simctl", "list", "runtimes"], check=False) + _dump_log() + sys.exit(1) + + device_type_id = _find_iphone_device_type() + if not device_type_id: + log("ERROR: No usable iPhone simulator device type found.", tee=True) + run_cmd(["xcrun", "simctl", "list", "devicetypes"], check=False) + _dump_log() + sys.exit(1) + + name = _unique_simulator_name() + log(f"Creating simulator: name='{name}' type={device_type_id} runtime={runtime_id}", + tee=True) + create = run_cmd( + ["xcrun", "simctl", "create", name, device_type_id, runtime_id], + check=False, + ) + if create.returncode != 0 or not create.stdout: + log(f"ERROR: simctl create failed (exit {create.returncode}).", tee=True) + _dump_log() + sys.exit(1) + + # `simctl create` prints just the UDID on stdout (one line). + udid = (create.stdout or "").strip().splitlines()[-1].strip() + if not re.match(r"^[0-9A-Fa-f-]{36}$", udid): + log(f"ERROR: simctl create produced unexpected output: {create.stdout!r}", + tee=True) + _dump_log() + sys.exit(1) + log(f"Created simulator UDID: {udid}", tee=True) + + log(f"Booting simulator {udid}", tee=True) + boot = run_cmd(["xcrun", "simctl", "boot", udid], check=False) + if boot.returncode != 0 and "Booted" not in (boot.stdout or ""): + # On shared Helix machines, even a fresh device can fail to boot + # because CoreSimulator writes log/state files into shared folders + # that may still have foreign ownership the chown didn't catch. + # Run another chown then retry once. If the retry still fails, give + # up — the diagnostics from chown should explain why. + log(f"WARNING: simctl boot failed (exit {boot.returncode}). " + f"Re-running CoreSimulator chown and retrying once.", tee=True) + fix_coresimulator_permissions() + boot = run_cmd(["xcrun", "simctl", "boot", udid], check=False) + if boot.returncode != 0 and "Booted" not in (boot.stdout or ""): + log(f"ERROR: simctl boot retry failed (exit {boot.returncode}).", + tee=True) + _dump_log() + sys.exit(1) + log("Boot retry succeeded after chown.", tee=True) + + # Wait for the simulator to finish booting. Without this, downstream + # actool / mlaunch race against the boot and intermittently fail. + log(f"Waiting for boot to complete (simctl bootstatus -b)", tee=True) + bootstatus = run_cmd(["xcrun", "simctl", "bootstatus", udid, "-b"], check=False) + if bootstatus.returncode != 0: + log(f"WARNING: bootstatus reported exit {bootstatus.returncode}; " + f"continuing anyway and relying on preflight.", tee=True) + + _simulator_preflight(udid) + _write_sim_udid(workitem_root, udid) + + log("Currently booted devices:") + run_cmd(["xcrun", "simctl", "list", "devices", "booted"], check=False) + return udid + + +def _run_workload_cmd(args, timeout_seconds): + """Run a workload install command with a timeout. + + Returns a CompletedProcess. If the command times out, kills the process + and returns a synthetic CompletedProcess with returncode=-1. + + The dotnet CLI can hang for hours when NuGet feeds are slow or broken + (internal download retries). A timeout ensures the fallback retry has + time to run within the Helix work item timeout. + """ + try: + return run_cmd(args, check=False, timeout=timeout_seconds) + except subprocess.TimeoutExpired as e: + log(f"WARNING: Command timed out after {timeout_seconds}s — killed", tee=True) + # Log tail of any partial output captured before the kill + partial = getattr(e, 'output', '') or '' + if partial: + if not isinstance(partial, str): + partial = partial.decode('utf-8', errors='replace') + for line in partial.splitlines()[-20:]: + log_raw(line) + return subprocess.CompletedProcess(args, returncode=-1) + + +# Cap each workload install attempt so there's time for the fallback retry +# within the Helix work item timeout (2:30). Without this, the dotnet CLI +# can hang for 2+ hours on NuGet download failures (internal retries). +_WORKLOAD_INSTALL_TIMEOUT = 1200 # 20 minutes per attempt + +# NuGet restore should complete much faster than workload install, but can +# still hang on dead feeds. 10 minutes is generous; prevents consuming the +# entire Helix work item timeout (2:30) on a hung restore. +_RESTORE_TIMEOUT = 600 # 10 minutes + + +def install_workload(ctx): + """Install the maui-ios workload using the shipped SDK. + + Uses the rollback file created by pre.py to pin to the exact workload + version (latest nightly packs). Falls back to a plain install if no + rollback file is present. Always uses --ignore-failed-sources because + dead NuGet feeds are common in CI. + + Each attempt is capped at _WORKLOAD_INSTALL_TIMEOUT seconds to prevent + slow NuGet downloads from consuming the entire Helix work item timeout. + + Manifest-patching dependency (pre.py): + When the iOS workload manifest references net10.0 cross-targeting + packs that don't exist on NuGet, pre.py patches the manifest to + remove those entries and places the patched files inside the SDK tree + at ``$DOTNET_ROOT/sdk-manifests/{band}/microsoft.net.sdk.ios/{ver}/``. + The SDK tree ships to Helix as the correlation payload, so the + patched manifest is already on disk. The ``--skip-manifest-update`` + retry below tells the CLI to use on-disk manifests instead of + downloading new ones, which picks up the patched manifest and only + installs packs that actually exist. + """ + log_raw("=== WORKLOAD INSTALL ===", tee=True) + + rollback_file = os.path.join(ctx["workitem_root"], "rollback_maui.json") + nuget_config = ctx["nuget_config"] + + install_args = [ + ctx["dotnet_exe"], "workload", "install", "maui-ios", + ] + + if os.path.isfile(rollback_file): + log(f"Using rollback file: {rollback_file}") + install_args.extend(["--from-rollback-file", rollback_file]) + else: + log("No rollback_maui.json found — installing latest maui-ios workload") + + if os.path.isfile(nuget_config): + install_args.extend(["--configfile", nuget_config]) + + # Dead NuGet feeds are common in CI — always tolerate failures + install_args.append("--ignore-failed-sources") + + result = _run_workload_cmd(install_args, _WORKLOAD_INSTALL_TIMEOUT) + if result.returncode != 0 and os.path.isfile(rollback_file): + # The --from-rollback-file attempt typically fails for one of two + # reasons: + # 1. NuGet version skew — packs referenced by a new manifest have + # not propagated to all NuGet feeds yet ("package NOT FOUND"). + # 2. net10.0 cross-targeting packs — the manifest references packs + # that don't exist on any feed (upstream coherency issue). + # + # In case (2), pre.py's manifest-patching fallback has already + # placed a patched manifest (with net10.0 entries removed) inside + # the SDK tree, which was shipped here as the correlation payload. + # --skip-manifest-update tells the CLI to use that on-disk manifest + # instead of downloading a new one, so it only resolves packs that + # actually exist. + log(f"WARNING: Workload install with rollback file failed " + f"(exit code {result.returncode})", tee=True) + log("Retrying with --skip-manifest-update (uses on-disk manifest " + "from correlation payload)...", tee=True) + + retry_args = [ + ctx["dotnet_exe"], "workload", "install", "maui-ios", + "--skip-manifest-update", + ] + if os.path.isfile(nuget_config): + retry_args.extend(["--configfile", nuget_config]) + retry_args.append("--ignore-failed-sources") + + result = _run_workload_cmd(retry_args, _WORKLOAD_INSTALL_TIMEOUT) + + if result.returncode != 0: + log(f"WORKLOAD INSTALL FAILED (exit code {result.returncode})", tee=True) + _dump_log() + sys.exit(1) + + log("maui-ios workload installed successfully") + + +def restore_packages(ctx): + """Restore NuGet packages for the app project. + + Uses --ignore-failed-sources and /p:NuGetAudit=false to handle dead + feeds and avoid audit warnings that slow down restore. + """ + log_raw("=== NUGET RESTORE ===", tee=True) + + csproj = ctx["csproj"] + if not os.path.isfile(csproj): + log(f"ERROR: Project file not found at {csproj}", tee=True) + _dump_log() + sys.exit(2) + + restore_args = [ + ctx["dotnet_exe"], "restore", csproj, + "--ignore-failed-sources", + "/p:NuGetAudit=false", + ] + + nuget_config = ctx["nuget_config"] + if os.path.isfile(nuget_config): + restore_args.extend(["--configfile", nuget_config]) + + framework = ctx.get("framework") + if framework: + restore_args.append(f"/p:TargetFrameworks={framework}") + + msbuild_args = ctx.get("msbuild_args") + if msbuild_args: + restore_args.extend(msbuild_args.split()) + + try: + result = run_cmd(restore_args, check=False, timeout=_RESTORE_TIMEOUT) + except subprocess.TimeoutExpired: + log(f"RESTORE TIMED OUT after {_RESTORE_TIMEOUT}s", tee=True) + _dump_log() + sys.exit(2) + if result.returncode != 0: + log(f"RESTORE FAILED (exit code {result.returncode})", tee=True) + _dump_log() + sys.exit(2) + + log("NuGet restore succeeded") + + +def disable_spotlight(workitem_root): + """Disable Spotlight indexing on the workitem directory. + + Spotlight's mds_stores process can hold file locks during builds, + causing intermittent build failures. This is a well-known issue on + macOS CI machines. + """ + log_raw("=== DISABLE SPOTLIGHT ===", tee=True) + result = run_cmd( + ["sudo", "mdutil", "-i", "off", workitem_root], + check=False, + ) + if result.returncode != 0: + # Non-fatal — Spotlight interference is intermittent + log(f"WARNING: mdutil -i off failed (exit {result.returncode}). " + "Spotlight may interfere with builds.", tee=True) + else: + log(f"Spotlight indexing disabled for {workitem_root}") + + +def detect_physical_device(): + """Detect whether a physical iOS device is connected and return its UDID. + + Checks IOS_DEVICE_UDID env var first, then uses 'xcrun devicectl list devices'. + Returns the UDID string, or None if no device is found. + + NOTE: This duplicates iOSHelper.detect_connected_device() intentionally. + setup_helix.py runs as a standalone Helix pre-command with minimal imports + (no performance.common, no shared.ioshelper). Keeping this self-contained + avoids import failures on the Helix machine. + """ + log_raw("=== PHYSICAL DEVICE DETECTION ===", tee=True) + + udid = os.environ.get("IOS_DEVICE_UDID", "").strip() + if udid: + log(f"Using IOS_DEVICE_UDID from environment: {udid}", tee=True) + return udid + + # Auto-detect via devicectl + result = run_cmd( + ["xcrun", "devicectl", "list", "devices"], + check=False, + ) + if result.returncode != 0: + log("WARNING: 'xcrun devicectl list devices' failed. " + "No physical device detection available.", tee=True) + return None + + # Log the full output for debugging + log("devicectl output:") + for line in (result.stdout or "").splitlines(): + log_raw(f" {line}") + + # Try JSON output for structured parsing + # Write to temp file instead of /dev/stdout because devicectl mixes + # human-readable table text and JSON when writing to stdout. + import tempfile + fd, json_tmp = tempfile.mkstemp(suffix='.json', prefix='devicectl_') + os.close(fd) + try: + json_result = run_cmd( + ["xcrun", "devicectl", "list", "devices", "--json-output", json_tmp], + check=False, + ) + if json_result.returncode == 0 and os.path.exists(json_tmp): + try: + import json + with open(json_tmp, 'r') as f: + data = json.load(f) + devices = data.get("result", {}).get("devices", []) + for device in devices: + conn = device.get("connectionProperties", {}) + transport = conn.get("transportType", "") + name = device.get("deviceProperties", {}).get("name", "unknown") + # Prefer hardware UDID (e.g. 00008020-001965D83C43002E) over + # CoreDevice identifier (a UUID). mlaunch requires the hardware + # UDID — same logic as iOSHelper.detect_connected_device(). + hw_udid = device.get("hardwareProperties", {}).get("udid", "") + device_udid = hw_udid or device.get("identifier", "") + if transport in ("wired", "localNetwork", "wifi") and device_udid: + log(f"Found connected device: {name} (UDID: {device_udid}, " + f"transport: {transport})", tee=True) + return device_udid + except Exception as e: + log(f"JSON parsing failed: {e}", tee=True) + finally: + if os.path.exists(json_tmp): + os.remove(json_tmp) + + log("No connected physical devices found.", tee=True) + return None + + +# --- Main --- + +# --- Device signing prerequisite check --- + +# Device signing replicates the Helix XHarness apple recipe: the provisioning +# profile is DOWNLOADED at sign time (ioshelper.sign_app_for_device), and the +# cert lives in this dedicated keychain, unlocked with the password file below. +# So the pre-build gate checks the keychain + password + a codesigning identity — +# NOT a profile on disk (it isn't installed there) and NOT a 'sign' tool (none +# exists on the perf Macs). Earlier disk/sign-tool searches were wrong and +# produced a false "DEVICE INFRA UNAVAILABLE". +_SIGNING_KEYCHAIN = "signing-certs.keychain-db" +_SIGNING_KEYCHAIN_PASSWORD_FILE = os.path.expanduser("~/.config/keychain") + + +def verify_signing_prerequisites(): + """Verify the device code-signing prerequisites are present. + + Checks the signing keychain (signing-certs.keychain-db), its password file + (~/.config/keychain), and that the keychain holds a valid codesigning + identity. The provisioning profile is downloaded at sign time, so it is not + checked here. A fast pre-build gate. Returns True iff all are present. + Does not raise. + """ + log_raw("=== DEVICE SIGNING PREREQUISITE CHECK ===", tee=True) + ok = True + + # Signing keychain present in the search list? + try: + listing = subprocess.run( + ["security", "list-keychains"], capture_output=True, text=True, + timeout=30).stdout or "" + except Exception as e: + listing = "" + log(f"WARNING: `security list-keychains` failed: {e}", tee=True) + if _SIGNING_KEYCHAIN in listing: + log(f"Found signing keychain: {_SIGNING_KEYCHAIN}", tee=True) + else: + log(f"WARNING: keychain '{_SIGNING_KEYCHAIN}' not in `security " + f"list-keychains`. Present: {listing.strip()}", tee=True) + ok = False + + # Keychain password file present? + if os.path.isfile(_SIGNING_KEYCHAIN_PASSWORD_FILE): + log(f"Found keychain password file: {_SIGNING_KEYCHAIN_PASSWORD_FILE}", tee=True) + else: + log(f"WARNING: keychain password file {_SIGNING_KEYCHAIN_PASSWORD_FILE} " + "not found", tee=True) + ok = False + + # A valid codesigning identity in that keychain? + try: + idout = subprocess.run( + ["security", "find-identity", "-vp", "codesigning", _SIGNING_KEYCHAIN], + capture_output=True, text=True, timeout=30).stdout or "" + log(f"security find-identity -vp codesigning {_SIGNING_KEYCHAIN}:\n{idout.strip()}") + if re.search(r"\b[1-9]\d* valid identities found", idout): + log("Found a valid codesigning identity in the signing keychain", tee=True) + else: + log("WARNING: no valid codesigning identity in the signing keychain", tee=True) + ok = False + except Exception as e: + log(f"WARNING: `security find-identity` failed: {e}", tee=True) + ok = False + + return ok + + +def main(): + global _logfile + + # Open log file in HELIX_WORKITEM_UPLOAD_ROOT for post-mortem debugging + upload_root = os.environ.get("HELIX_WORKITEM_UPLOAD_ROOT") + if upload_root: + os.makedirs(upload_root, exist_ok=True) + _logfile = open(os.path.join(upload_root, "output.log"), "a") + + workitem_root = os.environ.get("HELIX_WORKITEM_ROOT", ".") + correlation_payload = os.environ.get("HELIX_CORRELATION_PAYLOAD", ".") + + # Determine target device type from iOSRid env var (set by .proj). + # ios-arm64 → physical device, iossimulator-* → simulator + ios_rid = os.environ.get("IOS_RID", "iossimulator-x64") + is_physical_device = (ios_rid == "ios-arm64") + + # Detect host architecture to select the correct simulator RID. + # Mac.iPhone.17.Perf queue uses Intel x64 machines which need + # iossimulator-x64, not iossimulator-arm64. Apple Silicon needs + # iossimulator-arm64. Physical device builds (ios-arm64) target the + # iPhone hardware, not the Mac, so skip architecture override. + if not is_physical_device: + host_arch = platform.machine() + if host_arch == "x86_64": + ios_rid = "iossimulator-x64" + elif host_arch == "arm64": + ios_rid = "iossimulator-arm64" + else: + log(f"WARNING: Unknown architecture '{host_arch}', " + f"keeping IOS_RID={ios_rid}", tee=True) + os.environ["IOS_RID"] = ios_rid + log(f"Host architecture: {host_arch}, using IOS_RID={ios_rid}", tee=True) + + # The simulator device name env var is no longer used: we always create + # our own fresh simulator under this user's CoreSimulator namespace via + # create_and_boot_simulator(). Pre-existing devices on Helix machines + # may belong to other users and refuse to boot with permission errors. + + # Framework and MSBuild args are passed as command-line arguments when + # available (from the .proj PreCommands), or fall back to env vars. + framework = sys.argv[1] if len(sys.argv) > 1 else os.environ.get("PERFLAB_Framework", "") + msbuild_args = sys.argv[2] if len(sys.argv) > 2 else "" + + ctx = { + "framework": framework, + "msbuild_args": msbuild_args, + "workitem_root": workitem_root, + "correlation_payload": correlation_payload, + "nuget_config": os.path.join(workitem_root, "app", "NuGet.config"), + "csproj": os.path.join(workitem_root, "app", "MauiiOSInnerLoop.csproj"), + } + + log_raw("=== iOS HELIX SETUP START ===", tee=True) + log(f"Target device type: {'physical device' if is_physical_device else 'simulator'} " + f"(IOS_RID={ios_rid})", tee=True) + + # Step 1: Configure the .NET SDK from the correlation payload + ctx["dotnet_exe"] = setup_dotnet(correlation_payload) + print_diagnostics() + + # Step 2: Select the Xcode version matching the iOS SDK workload packs. + # The packs require a specific Xcode (e.g. 26.2.x packs need Xcode 26.2). + # Selecting early avoids wasting 20+ min on workload install before + # _ValidateXcodeVersion fails. + select_xcode(workitem_root) + + # Step 3 & 4: Device-type-specific setup. + # Both job types create + boot a fresh simulator under THIS user's + # CoreSimulator namespace. Pre-existing simulators on the Helix machine + # may belong to other users (root or previous tenants) and refuse to boot + # with permission errors. Even physical-device builds need a writable + # simulator booted because actool spawns AssetCatalogSimulatorAgent via + # CoreSimulator during 'dotnet build' for ios-arm64. + fix_coresimulator_permissions() + validate_simulator_runtimes() + create_and_boot_simulator(workitem_root) + + if is_physical_device: + # Order matters: detect the physical device FIRST, signing artifacts + # SECOND. Both gates check independent infra (one is hardware on the + # USB hub, the other is files in the keychain / on disk), and either + # can fail by itself. Detecting the device first means the log always + # shows what hardware the queue exposed regardless of signing state, + # which is what humans actually need to debug a red work item. + + # Detect and validate the connected physical device. We require one; + # there is NO fallback to the simulator on a device job. A green + # device job MUST mean device measurements ran on real hardware, + # never that we silently downgraded to a simulator. + device_udid = detect_physical_device() + if not device_udid: + reason = ( + "No physical iOS device detected on this Helix machine. " + "Device job requires a connected, paired iPhone — there is " + "NO simulator fallback by design. This is a queue " + "provisioning gap (device disconnected, not paired, or " + "queue capacity issue), not a scenario bug. Verify the " + "Mac.iPhone.13.Perf machine has its iPhone connected and " + "paired (xcrun devicectl list devices)." + ) + log_raw("=" * 70, tee=True) + log_raw("WORK ITEM FAILED — NO PHYSICAL DEVICE", tee=True) + log_raw("=" * 70, tee=True) + log(reason, tee=True) + log_raw("=" * 70, tee=True) + _dump_log() + sys.exit(1) + + # Log the detected UDID for diagnostics. Note: os.environ changes + # in this Python process do NOT persist to subsequent Helix commands + # (test.py, post.py). runner.py re-detects the device independently + # via iOSHelper.detect_connected_device(). + os.environ["IOS_DEVICE_UDID"] = device_udid + log(f"IOS_DEVICE_UDID detected: {device_udid}", tee=True) + + # Verify code-signing prerequisites (signing keychain + password + + # a codesigning identity). ioshelper.sign_app_for_device downloads the + # provisioning profile and does the actual codesign after the build. + signing_ready = verify_signing_prerequisites() + + # Without the signing keychain + identity, the post-build codesign (and + # therefore `xcrun devicectl device install app`) cannot succeed — + # devicectl rejects an unsigned bundle with "No code signature found". + # Fail loudly so a genuinely missing prerequisite shows up as a red + # build, not a green-with-hidden-skip. + if not signing_ready: + reason = ( + "Device code-signing prerequisites not available on this Helix " + f"machine: need the signing keychain '{_SIGNING_KEYCHAIN}', its " + f"password file {_SIGNING_KEYCHAIN_PASSWORD_FILE}, AND a valid " + "codesigning identity in that keychain. This is a queue " + "provisioning gap, not a scenario bug. Fix: ensure the " + "macos-signing-certs artifact installed the signing keychain + " + "Apple Development identity (same as Mac.iPhone.17.Perf)." + ) + log_raw("=" * 70, tee=True) + log_raw("WORK ITEM FAILED — DEVICE INFRA UNAVAILABLE", tee=True) + log_raw("=" * 70, tee=True) + log(reason, tee=True) + log_raw("=" * 70, tee=True) + _dump_log() + sys.exit(1) + log(f"IOS_DEVICE_UDID detected: {device_udid}", tee=True) + + # Step 5: Install the maui-ios workload + # Must happen BEFORE restore because restore needs workload packs + install_workload(ctx) + + # Step 6: Restore NuGet packages + restore_packages(ctx) + + # Step 7: Disable Spotlight indexing to prevent file-lock errors + disable_spotlight(workitem_root) + + log_raw("=== iOS HELIX SETUP SUCCEEDED ===", tee=True) + _dump_log() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/scenarios/mauiiosinnerloop/test.py b/src/scenarios/mauiiosinnerloop/test.py new file mode 100644 index 00000000000..a19a6cfabdb --- /dev/null +++ b/src/scenarios/mauiiosinnerloop/test.py @@ -0,0 +1,14 @@ +''' +MAUI iOS Inner Loop (Debug End-to-End) Time Measurement +Orchestrates first build-deploy-startup → file edit → incremental build-deploy-startup → parse binlogs and startup times. +''' +from shared.runner import TestTraits, Runner + +EXENAME = 'MauiiOSInnerLoop' + + +if __name__ == "__main__": + traits = TestTraits(exename=EXENAME, + guiapp='false', + ) + Runner(traits).run() diff --git a/src/scenarios/shared/const.py b/src/scenarios/shared/const.py index 3492887ff13..ea2c78bb927 100644 --- a/src/scenarios/shared/const.py +++ b/src/scenarios/shared/const.py @@ -19,6 +19,7 @@ ANDROIDINSTRUMENTATION = "androidinstrumentation" DEVICEPOWERCONSUMPTION = "devicepowerconsumption" BUILDTIME = "buildtime" +IOSINNERLOOP = "iosinnerloop" ANDROIDINNERLOOP = "androidinnerloop" SCENARIO_NAMES = {STARTUP: 'Startup', @@ -29,6 +30,7 @@ INNERLOOPMSBUILD: 'InnerLoopMsBuild', DOTNETWATCH: 'DotnetWatch', BUILDTIME: 'BuildTime', + IOSINNERLOOP: 'IOSInnerLoop', ANDROIDINNERLOOP: 'AndroidInnerLoop'} BINDIR = 'bin' diff --git a/src/scenarios/shared/ioshelper.py b/src/scenarios/shared/ioshelper.py new file mode 100644 index 00000000000..d1df80f0afd --- /dev/null +++ b/src/scenarios/shared/ioshelper.py @@ -0,0 +1,1100 @@ +"""iOS deploy and startup measurement helpers (simulator + physical device). + +This module is the single source of truth for "F5-style" install + launch +measurement on iOS. Two device kinds, two slightly different toolchains: + + ┌──────────┬──────────────────────────┬──────────────────────────────┐ + │ │ install │ cold-startup measurement │ + ├──────────┼──────────────────────────┼──────────────────────────────┤ + │ device │ xcrun devicectl device │ mlaunch --launchdev │ + │ │ install app (bundle is │ (returns PID immediately, │ + │ │ codesigned first by │ watchdog log captures the │ + │ │ sign_app_for_device) │ startup phases) │ + ├──────────┼──────────────────────────┼──────────────────────────────┤ + │ simulator│ mlaunch --installsim │ xcrun simctl launch │ + │ │ (matches what the IDE │ (NOT mlaunch --launchsim, │ + │ │ does during F5) │ see Note 1 below) │ + └──────────┴──────────────────────────┴──────────────────────────────┘ + +Per .NET iOS team guidance (Rolf Bjarne Kvinge), mlaunch is the canonical +tool the IDEs use during F5 — it handles ad-hoc signing, devicectl tunnel +setup, and stdout tunnelling. We use it wherever the IDE would. + +Note 1 — why simctl for simulator launch instead of mlaunch --launchsim: + mlaunch --launchsim blocks until the launched app exits (it tunnels app + stdout/stderr) and during testing on Apple Silicon Helix machines the + simulator transitioned from Booted → Shutdown silently during the call, + producing 180-second timeouts with no diagnostic output. simctl launch + is what mlaunch invokes internally for the actual launch step, so the + wall-clock measurement is equivalent — we just skip mlaunch's stdout + tunnel layer (we don't need it for measurement). See + ``measure_cold_startup`` for the implementation and the post-launch + stabilization check that confirms the launched PID survives. + +Note 2 — physical-device install requires code signing: + ``xcrun devicectl device install app`` refuses to install an unsigned + bundle (MIInstallerErrorDomain code 13, "No code signature found"). The + .proj builds with ``EnableCodeSigning=false`` to keep the build + deterministic on Helix, then ``sign_app_for_device`` signs the bundle by + replicating the Helix XHarness apple recipe: download the provisioning + profile from a blob, unlock ``signing-certs.keychain-db`` (password in + ``~/.config/keychain``), and deep-sign with ``/usr/bin/codesign --sign + "Apple Development" --keychain signing-certs.keychain-db`` using entitlements + extracted from the profile. (There is no ``sign`` tool on the perf Macs, and + ``embedded.mobileprovision`` is served from a blob / lives inside signed + bundles, not installed on disk — earlier assumptions to the contrary were + wrong.) setup_helix.py verifies the keychain + password + identity are + present before the build so a missing prerequisite fails fast. +""" + +import glob +import json +import os +import plistlib +import re +import subprocess +import shutil +import tempfile +import time +from datetime import datetime +from logging import getLogger + +from performance.common import RunCommand + + +# Device code-signing constants — these mirror the Helix XHarness apple signing +# recipe (.packages/microsoft.dotnet.helix.sdk/*/tools/xharness-runner/ +# {XHarnessRunner.props, xharness-runner.apple.sh}), which is the proven path on +# these queues. The provisioning profile is DOWNLOADED (not installed on disk): +# "NET_Apple_Development" is the profile *filename*, and {PLATFORM}=iOS for device. +_PROVISIONING_PROFILE_URL = ( + "https://netcorenativeassets.blob.core.windows.net/resource-packages/" + "external/macos/signing/NET_Apple_Development_iOS.mobileprovision" +) +# The certificate lives in this dedicated keychain (installed by Helix's +# macos-signing-certs), unlocked with the password in ~/.config/keychain. We +# select it explicitly so codesign uses the profile-matching identity, not an +# unrelated one from the default keychain search list. +_SIGNING_KEYCHAIN = "signing-certs.keychain-db" +_SIGNING_KEYCHAIN_PASSWORD_FILE = os.path.expanduser("~/.config/keychain") +_LOGIN_KEYCHAIN = os.path.expanduser("~/Library/Keychains/login.keychain-db") +# Partial identity name (matches "Apple Development: ()"), same +# as xharness-runner.apple.sh. Override with IOS_SIGNING_IDENTITY if needed. +_SIGNING_IDENTITY_NAME = os.environ.get("IOS_SIGNING_IDENTITY", "Apple Development") +# Apple certificate-chain anchors for the "Apple Development" cert. In the +# headless Helix session codesign fails "unable to build chain to self-signed +# root" even with the full chain present, so we import these into both the +# signing and login keychains AND add the root as a trusted code-signing anchor. +# Order: WWDR G3 (the cert's direct issuer) then Apple Root CA (self-signed root). +_SIGNING_CHAIN_CERT_URLS = [ + "https://www.apple.com/certificateauthority/AppleWWDRCAG3.cer", + "https://www.apple.com/appleca/AppleIncRootCertificate.cer", +] +# The old Apple WWDR intermediate (no G-series), expired 2023-02-07. Its lingering +# presence makes codesign build the chain through it (candidate matched by issuer +# name) and fail "unable to build chain to self-signed root" even though the valid +# WWDR G3 is installed (SecTrust/verify-cert correctly skip it by key-id, which is +# why verify-cert passes while codesign fails). The documented fix is to delete it. +# SHA-1 (verified against Apple's AppleWWDRCA.cer, notAfter 2023-02-07): +_EXPIRED_WWDR_SHA1 = "FF6797793A3CD798DC5B2ABEF56F73EDC9F83A64" + + +class iOSHelper: + """Unified helper for iOS simulator and physical device operations. + + Callers use the same API (setup_device, install_app, measure_cold_startup, + cleanup) regardless of device type. See module docstring for the + install/launch tooling matrix and the rationale behind it. + """ + + _mlaunch_path = None # resolved once, cached for the process + + def __init__(self): + self.bundle_id = None + self.device_id = None + self.app_bundle_path = None + self.is_physical_device = False + + # ── mlaunch Resolution ──────────────────────────────────────────── + + @staticmethod + def _resolve_mlaunch(): + """Resolve the mlaunch binary from the newest installed Microsoft.iOS.Sdk pack. + + Sorts candidates by the pack *version* directory (parent of `tools/`), + not by the pack name, so e.g. 26.10 ranks above 26.2. + """ + if iOSHelper._mlaunch_path is not None: + return iOSHelper._mlaunch_path + + dotnet_root = os.environ.get('DOTNET_ROOT', os.path.expanduser('~/.dotnet')) + pattern = os.path.join(dotnet_root, 'packs', 'Microsoft.iOS.Sdk.*', '*', 'tools', 'bin', 'mlaunch') + matches = glob.glob(pattern) + if not matches: + raise FileNotFoundError( + f"mlaunch not found. Searched: {pattern}\n" + f"Ensure the iOS SDK workload is installed (dotnet workload install ios)." + ) + + def _version_key(p: str): + # Path is .../packs///tools/bin/mlaunch — sort on . + version_dir = os.path.basename(os.path.dirname(os.path.dirname(os.path.dirname(p)))) + parts = re.split(r'[.\-+]', version_dir) + key = [] + for part in parts: + key.append((0, int(part)) if part.isdigit() else (1, part)) + return tuple(key) + + mlaunch = max(matches, key=_version_key) + getLogger().info("Resolved mlaunch: %s", mlaunch) + iOSHelper._mlaunch_path = mlaunch + return mlaunch + + # ── Device Detection ───────────────────────────────────────────── + + @staticmethod + def detect_connected_device(): + """Detect a connected physical iOS device and return its UDID. + + Checks IOS_DEVICE_UDID env var first, then auto-detects via devicectl. + Returns the UDID string, or None if no device is found. + """ + udid = os.environ.get('IOS_DEVICE_UDID', '').strip() + if udid: + getLogger().info("Using IOS_DEVICE_UDID from environment: %s", udid) + return udid + + getLogger().info("Auto-detecting connected iOS device...") + return iOSHelper._detect_via_devicectl_json() or iOSHelper._detect_via_devicectl_text() + + @staticmethod + def _detect_via_devicectl_json(): + """Try JSON-based device detection (Xcode 15+).""" + fd, json_tmp = tempfile.mkstemp(suffix='.json', prefix='devicectl_') + os.close(fd) + try: + result = subprocess.run( + ['xcrun', 'devicectl', 'list', 'devices', '--json-output', json_tmp], + capture_output=True, text=True, timeout=30 + ) + if result.returncode != 0: + return None + + with open(json_tmp, 'r') as f: + data = json.load(f) + + for device in data.get('result', {}).get('devices', []): + conn = device.get('connectionProperties', {}) + transport = conn.get('transportType', '') + # mlaunch uses the hardware UDID (e.g. 00008020-001965D83C43002E), + # NOT the CoreDevice identifier (a UUID like 5AE7F3E5-...). + # Fall back to the CoreDevice identifier if hw UDID is missing. + hw_udid = device.get('hardwareProperties', {}).get('udid', '') + device_udid = hw_udid or device.get('identifier', '') + if transport in ('wired', 'localNetwork', 'wifi') and device_udid: + name = device.get('deviceProperties', {}).get('name', 'unknown') + getLogger().info("Found device: %s (UDID: %s, transport: %s)", name, device_udid, transport) + return device_udid + return None + except (subprocess.TimeoutExpired, json.JSONDecodeError, KeyError, OSError): + return None + finally: + if os.path.exists(json_tmp): + os.remove(json_tmp) + + @staticmethod + def _detect_via_devicectl_text(): + """Fallback: parse device UDID from devicectl text output.""" + try: + result = subprocess.run( + ['xcrun', 'devicectl', 'list', 'devices'], + capture_output=True, text=True, timeout=30 + ) + if result.returncode != 0: + return None + + # Match CoreDevice UUID (8-4-4-4-12) or legacy Apple UDID formats + uuid_re = re.compile( + r'([0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}' + r'|[0-9A-Fa-f]{8}-[0-9A-Fa-f]{16}' + r'|[0-9A-Fa-f]{25,40})' + ) + for line in (result.stdout or '').splitlines(): + match = uuid_re.search(line) + if match: + getLogger().info("Detected device UDID: %s", match.group(1)) + return match.group(1) + return None + except Exception: + return None + + # ── Simulator UDID Resolution ──────────────────────────────────── + + @staticmethod + def _resolve_booted_simulator_udid(): + """Return the UDID of the simulator we should target. + + Resolution order: + 1. ``$HELIX_WORKITEM_ROOT/sim_udid.txt`` — written by setup_helix.py + after creating + booting our own per-workitem device. This pins + the UDID end-to-end so we never accidentally target a stale or + foreign device that happens to be booted on the machine. + 2. ``simctl list devices booted -j`` — fallback for local runs (no + Helix) where setup_helix.py was not invoked. + + mlaunch requires a real UDID (--device :v2:udid=); it does not + understand simctl's "booted" shortcut. + """ + # Prefer the pinned UDID written by setup_helix.py + workitem_root = os.environ.get('HELIX_WORKITEM_ROOT') + if workitem_root: + pinned = os.path.join(workitem_root, 'sim_udid.txt') + if os.path.isfile(pinned): + try: + with open(pinned, 'r', encoding='utf-8') as f: + udid = f.read().strip() + if udid: + getLogger().info("Using pinned simulator UDID from %s: %s", + pinned, udid) + return udid + except OSError as e: + getLogger().warning("Could not read %s: %s", pinned, e) + + try: + result = subprocess.run( + ['xcrun', 'simctl', 'list', 'devices', 'booted', '-j'], + capture_output=True, text=True, timeout=15 + ) + if result.returncode != 0: + return None + data = json.loads(result.stdout) + for runtime_devices in data.get('devices', {}).values(): + for dev in runtime_devices: + if dev.get('state', '').lower() == 'booted': + return dev['udid'] + return None + except Exception: + return None + + @staticmethod + def delete_simulator(udid): + """Shutdown and delete a simulator by UDID. Best-effort, never raises. + + Used by post.py to clean up the per-workitem simulator created in + setup_helix.py so devices don't accumulate across runs on the same + Helix machine. + """ + if not udid: + return + for cmd in (['xcrun', 'simctl', 'shutdown', udid], + ['xcrun', 'simctl', 'delete', udid]): + try: + subprocess.run(cmd, capture_output=True, text=True, timeout=60) + except Exception as e: + getLogger().warning("Cleanup '%s' failed: %s", ' '.join(cmd), e) + + # ── Unified Device Setup ───────────────────────────────────────── + + def setup_device(self, bundle_id, app_bundle_path, device_id='booted', is_physical=False): + """Prepare a simulator or physical device for testing. + + For simulators: ensures the device is booted and uninstalls any + existing app. For physical devices: stores the UDID for later use. + Does NOT install the app — call install_app() separately so + install timing can be captured independently. + """ + self.bundle_id = bundle_id + self.device_id = device_id + self.app_bundle_path = app_bundle_path + self.is_physical_device = is_physical + + if is_physical: + # Uninstall any stale app so first-deploy timing isn't affected + getLogger().info("Uninstalling any existing app (%s) from physical device: %s", bundle_id, device_id) + mlaunch = self._resolve_mlaunch() + self._run_quiet([mlaunch, '--uninstalldevbundleid', bundle_id, + '--devname', device_id]) + getLogger().info("Physical device setup complete: %s", device_id) + return + + # Simulator setup: boot if needed, uninstall stale app + if device_id != 'booted': + getLogger().info("Booting simulator: %s", device_id) + result = subprocess.run( + ['xcrun', 'simctl', 'boot', device_id], + capture_output=True, text=True + ) + if result.returncode != 0 and 'already booted' not in (result.stderr or '').lower(): + raise subprocess.CalledProcessError(result.returncode, result.args, result.stdout, result.stderr) + + # Resolve the actual UDID — mlaunch needs a real UDID, not "booted" + if self.device_id == 'booted': + resolved = self._resolve_booted_simulator_udid() + if not resolved: + raise RuntimeError( + "Could not resolve booted simulator UDID. " + "Ensure a simulator is booted (setup_helix.py should have done this).") + getLogger().info("Resolved booted simulator UDID: %s", resolved) + self.device_id = resolved + + self._run_quiet(['xcrun', 'simctl', 'uninstall', self.device_id, bundle_id]) + + # ── Device Code Signing ────────────────────────────────────────── + + def sign_app_for_device(self, app_bundle_path): + """Codesign the built .app bundle for physical-device deployment. + + Replicates the Helix XHarness apple signing recipe (the proven path on + these queues; see .packages/microsoft.dotnet.helix.sdk/*/tools/ + xharness-runner/xharness-runner.apple.sh). The build runs with + ``EnableCodeSigning=false`` so the bundle is unsigned, then here we: + + 1. Download the provisioning profile into the bundle as + ``embedded.mobileprovision`` (it is served from a blob, NOT + installed on disk — that earlier assumption was wrong). + 2. Unlock the ``signing-certs.keychain-db`` keychain. + 3. Extract the profile's ``Entitlements`` (``security cms -D``). + 4. Deep-sign every Mach-O / .app / .framework with + ``/usr/bin/codesign --sign "Apple Development" --keychain + signing-certs.keychain-db`` (deepest first; the top .app gets the + entitlements, nested code preserves metadata). + + The app's bundle id must fall under the profile's wildcard + (``net.dot.*``) for the install to be accepted on device. + + No-op for simulator builds or local runs (MSBuild signs those). + Raises on failure — a device job must fail loudly, never silently skip. + """ + if not self.is_physical_device: + return + + from performance.common import runninginlab + if not runninginlab(): + getLogger().info("Skipping post-build signing (local run — MSBuild handles signing)") + return + + # 1. Download the provisioning profile into the bundle. + profile_in_bundle = os.path.join(app_bundle_path, "embedded.mobileprovision") + getLogger().info("Downloading provisioning profile: %s", _PROVISIONING_PROFILE_URL) + RunCommand(["curl", "-sSL", "--fail", "--retry", "3", "-o", + profile_in_bundle, _PROVISIONING_PROFILE_URL], verbose=True).run() + if not os.path.isfile(profile_in_bundle) or os.path.getsize(profile_in_bundle) == 0: + raise RuntimeError( + f"Failed to download provisioning profile from {_PROVISIONING_PROFILE_URL}") + + # 2. Unlock the signing keychain, ensure the valid chain is present, and + # remove the expired 2023 WWDR intermediate that breaks chain-building. + self._unlock_signing_keychain() + self._ensure_signing_chain() + self._remove_expired_wwdr() + self._log_signing_diagnostics() + + # 3. Extract entitlements from the profile. + entitlements_path = self._extract_entitlements(profile_in_bundle) + + # 4. Deep-sign the bundle in-process (XHarness-style). The signing keychain + # is unlocked, codesign-authorized (set-key-partition-list) and set as + # the default with a pinned search list (see _unlock_signing_keychain). + self._codesign_bundle_deep(app_bundle_path, entitlements_path) + + # Verify so an install-time CoreSign rejection surfaces here with a clear + # message instead of an opaque devicectl error later. + verify = subprocess.run( + ["/usr/bin/codesign", "--verify", "--deep", "--strict", + "--verbose=2", app_bundle_path], + capture_output=True, text=True) + getLogger().info( + "codesign --verify exit %d\n%s", verify.returncode, + (verify.stderr or verify.stdout or "").strip()) + if verify.returncode != 0: + raise RuntimeError( + f"codesign verification failed (exit {verify.returncode}): " + f"{(verify.stderr or '').strip()}") + getLogger().info( + "Signed %s for device with identity '%s'", + os.path.basename(app_bundle_path), _SIGNING_IDENTITY_NAME) + + @staticmethod + def _unlock_signing_keychain(): + """Unlock the dedicated signing keychain (matches xharness-runner.apple.sh). + + The password lives in ~/.config/keychain. It is a secret, so it is NEVER + logged nor passed through a command line we echo. + """ + if not os.path.isfile(_SIGNING_KEYCHAIN_PASSWORD_FILE): + raise RuntimeError( + f"Keychain password file {_SIGNING_KEYCHAIN_PASSWORD_FILE} not " + f"found — cannot unlock {_SIGNING_KEYCHAIN}.") + listing = subprocess.run( + ["security", "list-keychains"], capture_output=True, text=True).stdout or "" + if _SIGNING_KEYCHAIN not in listing: + raise RuntimeError( + f"Keychain {_SIGNING_KEYCHAIN} not found in `security " + f"list-keychains`. Present: {listing.strip()}") + with open(_SIGNING_KEYCHAIN_PASSWORD_FILE) as f: + password = f.read().strip() + # Run directly (no RunCommand/verbose) so the password is not logged. + result = subprocess.run( + ["security", "unlock-keychain", "-p", password, _SIGNING_KEYCHAIN], + capture_output=True, text=True) + if result.returncode != 0: + raise RuntimeError( + f"Failed to unlock {_SIGNING_KEYCHAIN} (exit {result.returncode}): " + f"{(result.stderr or '').strip()}") + getLogger().info("Unlocked signing keychain %s", _SIGNING_KEYCHAIN) + # Authorize codesign/apple-tool to use the signing key without a GUI + # prompt. Without this partition-list (ACL) entry, codesign in a headless + # (non-GUI) Helix session fails with `errSecInternalComponent` even when + # the identity is present and the keychain is unlocked — the actual fatal + # error behind the misleading "unable to build chain to self-signed root" + # warning. This is the standard headless-macOS-CI signing fix. Password is + # never logged; only the outcome is. + part = subprocess.run( + ["security", "set-key-partition-list", "-S", + "apple-tool:,apple:,codesign:", "-s", "-k", password, _SIGNING_KEYCHAIN], + capture_output=True, text=True) + getLogger().info( + "set-key-partition-list on %s -> %d%s", _SIGNING_KEYCHAIN, + part.returncode, + "" if part.returncode == 0 else f": {(part.stderr or '').strip()[:200]}") + # Standard headless-macOS signing setup (last untried piece): make the + # signing keychain the DEFAULT and pin the session search list so codesign + # resolves the identity + its private key from it. Some codesign key + # operations consult the default keychain; without this they can fail + # errSecInternalComponent even with the keychain unlocked + authorized. + for setup in ( + ["security", "list-keychains", "-d", "user", "-s", _SIGNING_KEYCHAIN, + _LOGIN_KEYCHAIN, "/Library/Keychains/System.keychain"], + ["security", "default-keychain", "-s", _SIGNING_KEYCHAIN], + ): + s = subprocess.run(setup, capture_output=True, text=True) + getLogger().info("%s -> %d%s", " ".join(setup[:2]), s.returncode, + "" if s.returncode == 0 else f": {(s.stderr or '').strip()[:150]}") + + @staticmethod + def _ensure_signing_chain(): + """Make the Apple Development chain trustable to codesign. + + Import WWDR G3 + Apple Root CA into the signing + login keychains, then + (the key step, previously untried) mark the Apple Root as a TRUSTED + code-signing anchor in the ADMIN trust domain via ``sudo -n security + add-trusted-cert -d``. The user-domain form failed with "no user + interaction was possible"; the admin form uses root privilege and needs + no interaction. This directly addresses codesign's "unable to build chain + to self-signed root" (importing a root != trusting it as an anchor). All + best-effort; ``sudo -n`` fails fast (never hangs) if not permitted.""" + root_cert = None + for url in _SIGNING_CHAIN_CERT_URLS: + name = url.rsplit("/", 1)[-1] + dest = os.path.join(tempfile.gettempdir(), name) + try: + RunCommand(["curl", "-sSL", "--fail", "--retry", "3", "-o", dest, url], + verbose=True).run() + except Exception as e: + getLogger().warning("Failed to download %s: %s", name, e) + continue + if "Root" in name: + root_cert = dest + for keychain in (_SIGNING_KEYCHAIN, _LOGIN_KEYCHAIN): + result = subprocess.run(["security", "import", dest, "-k", keychain], + capture_output=True, text=True) + stderr = (result.stderr or "") + if result.returncode == 0: + getLogger().info("Imported %s into %s", name, keychain) + elif "already exists" in stderr: + getLogger().info("%s already present in %s", name, keychain) + else: + getLogger().info("Import %s into %s -> %d: %s", + name, keychain, result.returncode, stderr.strip()) + + # Trust the Apple Root as a code-signing anchor in the admin domain (root + # privilege, no user interaction). This is what makes codesign accept the + # self-signed root, which importing alone does not. + if root_cert: + # `add-trusted-cert -d` triggers the com.apple.trust-settings.admin + # authorization right, which prompts for a password even as root and + # so fails "no user interaction was possible" in the headless Helix + # session. Pre-authorize that right in the authorization DB (sudo) so + # the trust-settings write can proceed non-interactively. Best-effort; + # `sudo -n` fails fast if not permitted. + auth = subprocess.run( + ["sudo", "-n", "security", "authorizationdb", "write", + "com.apple.trust-settings.admin", "allow"], + capture_output=True, text=True) + getLogger().info( + "authorizationdb write trust-settings.admin allow -> %d: %s", + auth.returncode, (auth.stderr or auth.stdout or "").strip()[:200]) + for extra in (["-p", "codeSign"], []): # policy-scoped, then all-policy + try: + r = subprocess.run( + ["sudo", "-n", "security", "add-trusted-cert", "-d", + "-r", "trustRoot"] + extra + [root_cert], + capture_output=True, text=True, timeout=60) + getLogger().info( + "sudo add-trusted-cert -d %s -> %d: %s", + " ".join(extra) or "(all policies)", r.returncode, + (r.stderr or r.stdout or "").strip()[:300]) + if r.returncode == 0: + break + except subprocess.TimeoutExpired: + getLogger().warning("sudo add-trusted-cert timed out — continuing") + break + except Exception as e: + getLogger().warning("sudo add-trusted-cert failed: %s", e) + break + + @staticmethod + def _remove_expired_wwdr(): + """Delete the expired (2023-02-07) Apple WWDR intermediate so codesign + builds the chain through the valid WWDR G3 rather than the expired cert + ("unable to build chain to self-signed root"). Attempts the delete on + every candidate keychain unconditionally and logs each result — the + delete's own error ("could not be found" vs read-only vs success) + definitively locates the cert and whether it is removable. Best-effort.""" + candidates = [ + (None, False), # default search list + (_LOGIN_KEYCHAIN, False), + (os.path.expanduser("~/Library/Keychains/login.keychain"), False), + (_SIGNING_KEYCHAIN, False), + ("/Library/Keychains/System.keychain", True), + ] + for keychain, use_sudo in candidates: + label = keychain or "(search list)" + for _ in range(8): + cmd = (["sudo", "-n"] if use_sudo else []) + \ + ["security", "delete-certificate", "-Z", _EXPIRED_WWDR_SHA1] + if keychain: + cmd.append(keychain) + r = subprocess.run(cmd, capture_output=True, text=True) + msg = (r.stderr or r.stdout or "").strip()[:200] + getLogger().info("delete expired WWDR from %s -> %d: %s", + label, r.returncode, msg or "(ok)") + if r.returncode != 0: + break + # Verify the expired cert is actually gone from the search list (codesign + # picks it by issuer name, so any remaining copy re-breaks the chain). + check = subprocess.run( + ["security", "find-certificate", "-a", "-Z", + "-c", "Apple Worldwide Developer Relations Certification Authority"], + capture_output=True, text=True) + still_present = _EXPIRED_WWDR_SHA1 in (check.stdout or "") + getLogger().info("Expired WWDR still present after delete: %s", + still_present) + + @staticmethod + def _log_signing_diagnostics(): + """Log the keychain search list, WWDR/Root cert presence + validity, and + the admin trust settings to diagnose codesign 'unable to build chain to + self-signed root' trust failures (e.g. an expired intermediate, or the + anchor not being trusted).""" + for cmd in ( + ["security", "list-keychains"], + # -p prints PEM so we can check notAfter (expiry) of any duplicate/stale + # WWDR intermediates across the whole search list, not just one keychain. + ["security", "find-certificate", "-a", "-p", + "-c", "Apple Worldwide Developer Relations Certification Authority"], + ["sudo", "-n", "security", "dump-trust-settings", "-d"], + ): + try: + r = subprocess.run(cmd, capture_output=True, text=True, timeout=30) + out = (r.stdout or "") + if "find-certificate" in cmd and "BEGIN CERTIFICATE" in out: + # Summarize each PEM cert's subject + validity instead of raw PEM. + summary = [] + for block in out.split("-----END CERTIFICATE-----"): + pem = block + "-----END CERTIFICATE-----" + if "BEGIN CERTIFICATE" not in pem: + continue + info = subprocess.run( + ["openssl", "x509", "-noout", "-subject", "-enddate", + "-fingerprint", "-sha1"], + input=pem, capture_output=True, text=True) + summary.append((info.stdout or "").strip()) + out = "\n".join(summary) + getLogger().info("$ %s\n%s", " ".join(cmd), + (out or r.stderr or "").strip()[:1500]) + except Exception as e: + getLogger().warning("diagnostic `%s` failed: %s", " ".join(cmd), e) + + # Definitive SecTrust verdict on the signing leaf — codesign only reports a + # terse errSecInternalComponent. Extract the Apple Development leaf and run + # `security verify-cert -p codeSign`, which prints the EXACT chain/anchor + # result (untrusted root vs missing intermediate vs revoked...). -L stays + # offline (no revocation network calls that could themselves fail). + try: + leaf = subprocess.run( + ["security", "find-certificate", "-a", "-c", "Apple Development", + "-p", _SIGNING_KEYCHAIN], capture_output=True, text=True).stdout or "" + first = leaf.split("-----END CERTIFICATE-----")[0] + if "BEGIN CERTIFICATE" in first: + leaf_pem = os.path.join(tempfile.gettempdir(), "leaf_dev.pem") + with open(leaf_pem, "w") as f: + f.write(first + "-----END CERTIFICATE-----\n") + for vc in ( + ["security", "verify-cert", "-p", "codeSign", "-L", "-c", leaf_pem], + ["security", "verify-cert", "-p", "codeSign", "-c", leaf_pem], + ): + r = subprocess.run(vc, capture_output=True, text=True, timeout=60) + getLogger().info("$ %s -> %d\n%s", " ".join(vc), r.returncode, + (r.stdout or r.stderr or "").strip()[:800]) + except Exception as e: + getLogger().warning("verify-cert diagnostic failed: %s", e) + # Confirm the Apple Root CA anchor is actually present in the immutable + # system root store (SecTrust always consults it for anchors). + try: + r = subprocess.run( + ["security", "find-certificate", "-c", "Apple Root CA", + "/System/Library/Keychains/SystemRootCertificates.keychain"], + capture_output=True, text=True, timeout=30) + getLogger().info("Apple Root CA in SystemRootCertificates.keychain -> %d", + r.returncode) + except Exception as e: + getLogger().warning("system-root anchor check failed: %s", e) + + # Quick isolation probe: codesign a throwaway Mach-O with the same identity. + # If it fails errSecInternalComponent app-independently (while verify-cert + + # find-identity -v pass), the failure is the identity/session/keychain setup, + # not the app bundle. Runs after the default-keychain + partition-list setup. + try: + probe = os.path.join(tempfile.gettempdir(), "cs_probe.bin") + shutil.copyfile("/bin/ls", probe) + os.chmod(probe, 0o755) + r = subprocess.run( + ["/usr/bin/codesign", "-f", "--verbose=4", + "--sign", _SIGNING_IDENTITY_NAME, probe], + capture_output=True, text=True, timeout=60) + getLogger().info("codesign probe (in-process) -> %d\n%s", r.returncode, + (r.stderr or r.stdout or "").strip()[:600]) + except Exception as e: + getLogger().warning("codesign probe failed: %s", e) + + @classmethod + def _codesign_bundle_deep(cls, app_bundle_path, entitlements_path): + """Deep-sign like xharness-runner.apple.sh: sign every Mach-O / .app / + .framework deepest-first (the top .app gets the entitlements; nested code + preserves metadata). On the first failure re-run codesign ``--verbose=4`` + and log the full trust-evaluation detail. + + NOT passing ``--keychain`` — codesign finds the identity via the pinned + default search list (signing-certs is the default keychain, see + _unlock_signing_keychain) and anchors to the system-trusted Apple Root, + mirroring the ``verify-cert -p codeSign`` that passes on this leaf.""" + entries = [app_bundle_path] + for root, dirs, files in os.walk(app_bundle_path): + for name in files + dirs: + entries.append(os.path.join(root, name)) + entries.sort(key=lambda p: p.count(os.sep), reverse=True) + + signed = 0 + for path in entries: + ext = os.path.splitext(path)[1] + if not ((ext in (".app", ".framework") and os.path.isdir(path)) + or cls._is_macho(path)): + continue + cmd = ["/usr/bin/codesign", "-v", "--force", "--timestamp=none", + "--sign", _SIGNING_IDENTITY_NAME] + cmd += (["--entitlements", entitlements_path] if path == app_bundle_path + else ["--preserve-metadata=identifier,entitlements,flags"]) + cmd.append(path) + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + # Capture the full trust-evaluation detail once, to pinpoint the + # exact chain/anchor step codesign rejects. + diag = subprocess.run( + ["/usr/bin/codesign", "--verbose=4", "--force", + "--timestamp=none", "--sign", _SIGNING_IDENTITY_NAME, + "--preserve-metadata=identifier,entitlements,flags", path], + capture_output=True, text=True) + getLogger().error("codesign --verbose=4 detail:\n%s", + (diag.stderr or diag.stdout or "").strip()) + raise RuntimeError( + f"codesign failed for {path} (exit {result.returncode}) with " + f"identity '{_SIGNING_IDENTITY_NAME}': {(result.stderr or '').strip()}") + signed += 1 + getLogger().info("Deep-signed %d bundle component(s) in %s", + signed, os.path.basename(app_bundle_path)) + + @staticmethod + def _is_macho(path): + """True if ``path`` is a Mach-O binary (per ``file -b``).""" + if not os.path.isfile(path): + return False + try: + out = subprocess.run( + ["file", "-b", path], capture_output=True, text=True).stdout or "" + return "Mach-O" in out + except Exception: + return False + + @staticmethod + def _extract_entitlements(profile_path): + """Decode a provisioning profile and write its Entitlements to a temp + plist that ``codesign --entitlements`` consumes. + + ``security cms -D -i `` outputs the profile as an XML plist; + its ``Entitlements`` dict is what codesign needs. + """ + decoded = subprocess.run( + ["security", "cms", "-D", "-i", profile_path], + capture_output=True) + if decoded.returncode != 0: + raise RuntimeError( + f"Failed to decode provisioning profile {profile_path}: " + f"{decoded.stderr.decode(errors='replace').strip()}") + try: + profile_plist = plistlib.loads(decoded.stdout) + entitlements = profile_plist["Entitlements"] + except Exception as e: + raise RuntimeError( + f"Could not extract Entitlements from {profile_path}: {e}") + fd, ent_path = tempfile.mkstemp(suffix=".plist", prefix="entitlements_") + with os.fdopen(fd, "wb") as f: + plistlib.dump(entitlements, f) + getLogger().info( + "Extracted %d entitlement key(s) to %s", len(entitlements), ent_path) + return ent_path + + # ── Unified Operations ─────────────────────────────────────────── + + def install_app(self, app_bundle_path): + """Install the app bundle and return wall-clock install time in ms. + + Device: xcrun devicectl device install app --device + (the bundle is codesigned by sign_app_for_device first, so + devicectl's CoreSign requirement is satisfied). + Simulator: mlaunch --installsim --device :v2:udid= + (matches what the IDE does during F5). + """ + start = time.time() + + if self.is_physical_device: + cmd = ['xcrun', 'devicectl', 'device', 'install', 'app', + '--device', self.device_id, app_bundle_path] + else: + mlaunch = self._resolve_mlaunch() + cmd = [mlaunch, '--installsim', app_bundle_path, + '--device', f':v2:udid={self.device_id}'] + RunCommand(cmd, verbose=True).run() + + elapsed_ms = (time.time() - start) * 1000 + getLogger().info("Install completed in %.1f ms", elapsed_ms) + return elapsed_ms + + def measure_cold_startup(self, bundle_id): + """Measure app cold startup time in ms (int). + + - Simulator: ``xcrun simctl launch`` returns the launched PID + immediately. We measure wall-clock from invocation to PID + return (the same as what mlaunch internally reports), then + verify the process is still alive after a short stabilization + window so we don't report success for a crashed launch. + - Device: mlaunch --launchdev (returns immediately with PID) + + Note on the simulator path: an earlier version used + ``mlaunch --launchsim`` to better mirror the IDE F5 experience, + but on Apple Silicon Helix queues the simulator went from Booted + to Shutdown during/after that call (with no diagnostic output + from mlaunch). ``simctl launch`` is what mlaunch invokes + internally for the actual launch step, so the measurement is + equivalent for our purposes. + """ + + if self.is_physical_device: + return self._measure_device_startup_via_watchdog(bundle_id) + + # ── Simulator ──────────────────────────────────────────────── + # Sanity-check the simulator state before we start the timer so + # we fail fast with a clear error if the simulator has shut down + # (e.g. due to a previous workitem cleanup or system pressure). + self._assert_simulator_booted() + + # Verify the app is actually installed before timing the launch + # — otherwise an install-registration failure would be reported + # as a launch failure. + try: + container = subprocess.run( + ['xcrun', 'simctl', 'get_app_container', + self.device_id, bundle_id, 'app'], + capture_output=True, text=True, timeout=15, + ) + if container.returncode != 0: + raise RuntimeError( + f"App {bundle_id} is not installed in simulator " + f"{self.device_id} (simctl get_app_container exit " + f"{container.returncode}): {(container.stderr or '').strip()}" + ) + getLogger().info("App container: %s", (container.stdout or '').strip()) + except subprocess.TimeoutExpired: + raise RuntimeError( + f"simctl get_app_container timed out — simulator " + f"{self.device_id} not responding") + + # Terminate any running instance for a true cold start + self._run_quiet(['xcrun', 'simctl', 'terminate', self.device_id, bundle_id]) + time.sleep(0.5) + + # `simctl launch` returns immediately with the launched PID. + # `--terminate-running-process` makes the launch reliably cold + # even if termination above didn't take effect. + cmd = ['xcrun', 'simctl', 'launch', '--terminate-running-process', + self.device_id, bundle_id] + getLogger().info("$ %s", ' '.join(cmd)) + start = time.time() + result = subprocess.run(cmd, capture_output=True, text=True, timeout=180) + elapsed_ms = int((time.time() - start) * 1000) + + if result.returncode != 0: + # Dump diagnostics so we can tell whether the simulator died + # or the app failed to launch. + self._dump_simulator_diagnostics(bundle_id) + raise subprocess.CalledProcessError( + result.returncode, cmd, result.stdout, result.stderr) + + # `simctl launch` prints ": " on success. + pid = None + for token in (result.stdout or '').split(): + if token.isdigit(): + pid = int(token) + break + if pid is None: + getLogger().warning( + "Could not parse PID from simctl launch output: %r", + (result.stdout or '').strip()) + else: + getLogger().info("Launched %s with PID %d", bundle_id, pid) + # Stabilization check: the app should still be alive 2s + # later. If not, the launch was a crash, not a real start. + # + # iOS Simulator apps run as real macOS processes (sandboxed + # but in the host's process table — the PID returned by + # `simctl launch` IS the host PID), so we use host `ps -p` + # for the check. The simulator's userland does NOT include + # /bin/ps, so `simctl spawn ps` would fail with + # ENOENT regardless of whether the app is alive. + time.sleep(2.0) + check = subprocess.run( + ['ps', '-p', str(pid), '-o', 'pid='], + capture_output=True, text=True, timeout=15, + ) + if check.returncode != 0 or str(pid) not in (check.stdout or ''): + self._dump_simulator_diagnostics(bundle_id) + raise RuntimeError( + f"App {bundle_id} (PID {pid}) crashed within 2s of " + f"launch; host ps -p exit " + f"{check.returncode}, output: " + f"{(check.stdout or '').strip()!r}") + + getLogger().info("Cold startup: %d ms", elapsed_ms) + return elapsed_ms + + def _assert_simulator_booted(self): + """Raise RuntimeError if ``self.device_id`` is not in the Booted state.""" + try: + result = subprocess.run( + ['xcrun', 'simctl', 'list', 'devices', '-j'], + capture_output=True, text=True, timeout=15, + ) + except subprocess.TimeoutExpired: + raise RuntimeError("simctl list devices timed out") + try: + data = json.loads(result.stdout or '{}') + except Exception: + data = {} + for runtime_devices in (data.get('devices') or {}).values(): + for dev in runtime_devices: + if dev.get('udid') == self.device_id: + state = dev.get('state', '?') + if state != 'Booted': + raise RuntimeError( + f"Simulator {self.device_id} is in state " + f"{state!r}, expected 'Booted'. " + f"It must have been shut down between " + f"create_and_boot_simulator and the measurement.") + return + raise RuntimeError( + f"Simulator {self.device_id} not found in `simctl list devices`.") + + def _dump_simulator_diagnostics(self, bundle_id): + """Best-effort diagnostics dump for a failed simulator launch.""" + for cmd in ( + ['xcrun', 'simctl', 'list', 'devices', self.device_id], + ['xcrun', 'simctl', 'listapps', self.device_id], + # iOS sim has no /bin/ps; query the host process table for + # any descendants of the simulator hosting the bundle id. + ['ps', '-A', '-o', 'pid,ppid,command'], + ): + try: + r = subprocess.run(cmd, capture_output=True, text=True, timeout=15) + getLogger().error( + "$ %s\n--- exit %d ---\n%s\n--- stderr ---\n%s", + ' '.join(cmd), r.returncode, + (r.stdout or '')[-3000:], (r.stderr or '')[-1500:]) + except Exception as e: + getLogger().error("Diagnostic %s failed: %s", ' '.join(cmd), e) + + def _measure_device_startup_via_watchdog(self, bundle_id): + """Measure physical device cold startup using SpringBoard watchdog events. + + During every iOS app launch, SpringBoard emits four watchdog log events: + 1. "Now monitoring resource allowance of 20.00s" — OS starts loading the process + 2. "Stopped monitoring" — app reached main() + 3. "Now monitoring resource allowance of N.NNs" — OS waits for first frame + 4. "Stopped monitoring" — first frame drawn + + Time to Main = event2.timestamp - event1.timestamp + Time to First Draw = event4.timestamp - event3.timestamp + Total startup = Time to Main + Time to First Draw + + Requires sudo for `log collect --device`. + """ + # Give device a moment to settle before launch + time.sleep(1) + + # Record timestamp before launch for log collection window + start_ts = time.strftime('%Y-%m-%d %H:%M:%S%z') + + # Launch the app via mlaunch --launchdev (Rolf's guidance — same + # tool the IDEs use during F5; consistent with --installdev above). + # mlaunch --launchdev BLOCKS until the app exits (it tunnels stdout/ + # stderr from the device), so run it in a subprocess and terminate + # after we've collected enough log data. Killing mlaunch closes the + # tunnel but the app stays running on the device long enough for + # SpringBoard to emit the watchdog events we need. + mlaunch = self._resolve_mlaunch() + launch_cmd = [mlaunch, '--launchdev', self.app_bundle_path, + '--devname', self.device_id] + getLogger().info("$ %s", ' '.join(launch_cmd)) + launch_proc = subprocess.Popen( + launch_cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + try: + # Wait for the app to fully start before collecting logs. + time.sleep(5) + finally: + launch_proc.terminate() + try: + launch_proc.wait(timeout=5) + except subprocess.TimeoutExpired: + launch_proc.kill() + launch_proc.wait() + + # Collect device logs covering the launch window. + # Use a unique per-call logarchive path so we don't need to delete a + # prior root-owned archive (the upstream code uses `sudo rm -rf` to + # purge a stale shared path; that requires interactive sudo when + # NOPASSWD is scoped only to `log collect`, which hangs background + # campaigns). Per-call uniqueness eliminates the need for cleanup + # entirely; macOS's tmp cleaner reaps leaked archives. + import time as _time + logarchive = os.path.join( + tempfile.gettempdir(), + f'ioshelper_startup_{os.getpid()}_{int(_time.time() * 1000)}.logarchive') + collect_cmd = ['sudo', 'log', 'collect', '--device', + '--start', start_ts, '--output', logarchive] + RunCommand(collect_cmd, verbose=True).run() + + # Parse SpringBoard watchdog events for this bundle ID. + # sudo because the logarchive is root-owned (created by sudo log collect). + show_cmd = ['sudo', 'log', 'show', + '--predicate', '(process == "SpringBoard") && (category == "Watchdog")', + '--info', '--style', 'ndjson', logarchive] + show = RunCommand(show_cmd, verbose=True) + show.run() + + events = [] + for line in show.stdout.splitlines(): + try: + data = json.loads(line) + msg = data.get('eventMessage', '') + if bundle_id not in msg: + continue + if 'Now monitoring resource allowance' in msg or 'Stopped monitoring' in msg: + events.append(data) + except (json.JSONDecodeError, KeyError): + continue + + if len(events) < 4: + getLogger().warning("Expected 4 watchdog events, got %d — falling back to wall-clock", len(events)) + # Couldn't parse watchdog events; return -1 to signal invalid measurement + return -1 + + # Sort by timestamp — log show output is not guaranteed chronological + events.sort(key=lambda evt: evt['timestamp']) + + # Validate expected sequence: monitor → stop → monitor → stop + expected = ['Now monitoring', 'Stopped monitoring', 'Now monitoring', 'Stopped monitoring'] + for i, keyword in enumerate(expected): + if keyword not in events[i].get('eventMessage', ''): + getLogger().warning("Unexpected watchdog event sequence at index %d: %s", i, events[i].get('eventMessage', '')) + return -1 + + # Parse timestamps: "2026-04-13 20:36:19.836430+0200" + def parse_ts(evt): + return datetime.strptime(evt['timestamp'], '%Y-%m-%d %H:%M:%S.%f%z') + + t_main_start = parse_ts(events[0]) + t_main_end = parse_ts(events[1]) + t_draw_start = parse_ts(events[2]) + t_draw_end = parse_ts(events[3]) + + time_to_main_ms = int((t_main_end - t_main_start).total_seconds() * 1000) + time_to_draw_ms = int((t_draw_end - t_draw_start).total_seconds() * 1000) + total_ms = time_to_main_ms + time_to_draw_ms + + getLogger().info("Cold startup: %d ms (Time to Main: %d ms, Time to First Draw: %d ms)", + total_ms, time_to_main_ms, time_to_draw_ms) + + # Skip post-iter cleanup: the logarchive uses a unique per-call name so + # accumulation isn't a correctness problem. macOS reaps /tmp on a + # schedule. Leaving it avoids needing sudo (which hangs background + # campaigns when NOPASSWD is scoped only to `log collect`). + + return total_ms + + def cleanup(self, skip_uninstall=False): + """Clean up the device session (simulator or physical). + + Device uses mlaunch --uninstalldevbundleid. Simulator keeps simctl + (mlaunch has no simulator terminate/uninstall commands). + """ + if skip_uninstall: + return + if self.is_physical_device: + mlaunch = self._resolve_mlaunch() + self._run_quiet([mlaunch, '--uninstalldevbundleid', self.bundle_id, + '--devname', self.device_id]) + else: + self._run_quiet(['xcrun', 'simctl', 'terminate', self.device_id, self.bundle_id]) + self._run_quiet(['xcrun', 'simctl', 'uninstall', self.device_id, self.bundle_id]) + + # ── App Bundle Discovery ───────────────────────────────────────── + + def find_app_bundle(self, build_output_dir, app_name, configuration='Debug', is_physical=False): + """Find the .app bundle in the build output directory. + + Searches for: bin//net*//.app + Returns the absolute path. Raises FileNotFoundError if not found. + """ + rid_patterns = ['ios-arm64'] if is_physical else ['iossimulator-*'] + for rid_pattern in rid_patterns: + pattern = os.path.join(build_output_dir, 'bin', configuration, 'net*', rid_pattern, f'{app_name}.app') + matches = glob.glob(pattern) + if matches: + if len(matches) > 1: + getLogger().warning("Multiple app bundles found: %s. Using first.", matches) + app_path = os.path.abspath(matches[0]) + getLogger().info("Found app bundle: %s", app_path) + return app_path + + raise FileNotFoundError( + f"No .app bundle in {build_output_dir}/bin/{configuration}/net*/{rid_patterns[0]}/{app_name}.app" + ) + + # ── Helpers ─────────────────────────────────────────────────────── + + @staticmethod + def _run_quiet(cmd): + """Run a command, suppressing CalledProcessError (best-effort).""" + try: + RunCommand(cmd, verbose=True).run() + except subprocess.CalledProcessError: + pass diff --git a/src/scenarios/shared/precommands.py b/src/scenarios/shared/precommands.py index a7b6b7547a2..60491695f68 100644 --- a/src/scenarios/shared/precommands.py +++ b/src/scenarios/shared/precommands.py @@ -312,7 +312,14 @@ def get_packages_for_sdk_from_feed(self, sdk_name: str, feed: str): getLogger().debug(f"Raw packages response for {sdk_name}: {result.stdout}") try: - parsed_response = json.loads(result.stdout) + # Newer SDK versions (e.g., preview.4) may emit warnings or progress text + # before/after the JSON object in stdout. Extract only the JSON portion. + stdout_text = result.stdout + json_start = stdout_text.find('{') + json_end = stdout_text.rfind('}') + if json_start != -1 and json_end != -1 and json_end > json_start: + stdout_text = stdout_text[json_start:json_end + 1] + parsed_response = json.loads(stdout_text) getLogger().debug(f"Parsed JSON response for {sdk_name}: {parsed_response}") if not parsed_response.get("searchResult") or len(parsed_response["searchResult"]) == 0: diff --git a/src/scenarios/shared/runner.py b/src/scenarios/shared/runner.py index de069aab10d..fdf4c0059f0 100644 --- a/src/scenarios/shared/runner.py +++ b/src/scenarios/shared/runner.py @@ -18,6 +18,7 @@ from typing import Optional from shared.androidhelper import AndroidHelper from shared.androidinstrumentation import AndroidInstrumentationHelper +from shared.ioshelper import iOSHelper from shared.devicepowerconsumption import DevicePowerConsumptionHelper from shared.crossgen import CrossgenArguments from shared.startup import StartupWrapper @@ -32,6 +33,58 @@ from subprocess import CalledProcessError +def _make_counter(name, metric, results, top=False): + """Create a performance counter dict for JSON reports.""" + return { + "name": name, + "topCounter": top, + "defaultCounter": False, + "higherIsBetter": False, + "metricName": metric, + "results": results, + } + + +def _merge_deploy_report(build_report_path, install_results, startup_results, output_path, app_size_bytes=None): + """Merge build metrics with install/startup timing into a single E2E report. + + If the build report doesn't exist (local runs without PERFLAB_INLAB=1), + creates a minimal report with only install and startup counters. + """ + if os.path.exists(build_report_path): + with open(build_report_path, 'r') as f: + report = json.load(f) + if not report.get("tests"): + report["tests"] = [{"counters": []}] + elif "counters" not in report["tests"][0]: + report["tests"][0]["counters"] = [] + else: + report = {"tests": [{"counters": []}]} + + report["tests"][0]["counters"].append(_make_counter("Install Time", "ms", install_results)) + report["tests"][0]["counters"].append(_make_counter("Cold Startup Time", "ms", startup_results, top=True)) + if app_size_bytes is not None: + report["tests"][0]["counters"].append(_make_counter("App Bundle Size", "bytes", [app_size_bytes], top=True)) + + with open(output_path, 'w') as f: + json.dump(report, f, indent=2) + + +def _measure_app_size(app_bundle_path): + """Return the total size of a .app bundle directory in bytes. + + Lightweight inline equivalent of the SizeOnDisk tool (SizeOnDisk.cs / sod.py) + which is designed for standalone size-focused scenarios with per-file breakdowns. + This helper just needs the total for a single counter in the E2E report. + """ + total = 0 + for dirpath, _dirnames, filenames in os.walk(app_bundle_path): + for f in filenames: + fp = os.path.join(dirpath, f) + if not os.path.islink(fp): + total += os.path.getsize(fp) + return total + class Runner: ''' Wrapper for running all the things @@ -175,6 +228,20 @@ def parseargs(self): buildtimeparser.add_argument('--binlog-path', help='Location of binlog', dest='binlogpath') self.add_common_arguments(buildtimeparser) + iosinnerloopparser = subparsers.add_parser(const.IOSINNERLOOP, + description='measure first and incremental build+deploy time via binlogs (iOS)') + iosinnerloopparser.add_argument('--csproj-path', help='Path to .csproj file to build', dest='csprojpath') + iosinnerloopparser.add_argument('--edit-src', help='Modified source file paths, semicolon-separated', dest='editsrc') + iosinnerloopparser.add_argument('--edit-dest', help='Destination paths for modified files, semicolon-separated', dest='editdest') + iosinnerloopparser.add_argument('--framework', '-f', help='Target framework (e.g., net11.0-ios)', dest='framework') + iosinnerloopparser.add_argument('--configuration', '-c', help='Build configuration', dest='configuration', default='Debug') + iosinnerloopparser.add_argument('--msbuild-args', help='Additional MSBuild arguments', dest='msbuildargs', default='') + iosinnerloopparser.add_argument('--bundle-id', help='iOS bundle identifier', dest='bundleid') + iosinnerloopparser.add_argument('--device-id', help='iOS device ID (UDID for physical device, simulator ID or "booted" for simulator)', dest='deviceid', default='booted') + iosinnerloopparser.add_argument('--device-type', choices=['simulator', 'device'], help='Target device type: simulator (default) or physical device. Auto-detected from RuntimeIdentifier if not set.', dest='devicetype', default=None) + iosinnerloopparser.add_argument('--inner-loop-iterations', help='Number of incremental build+deploy+startup iterations (1+)', type=int, default=10, dest='innerloopiterations') + self.add_common_arguments(iosinnerloopparser) + androidinnerloopparser = subparsers.add_parser(const.ANDROIDINNERLOOP, description='measure first and incremental deploy time via binlogs') androidinnerloopparser.add_argument('--csproj-path', help='Path to .csproj file to build', dest='csprojpath', required=True) @@ -211,6 +278,33 @@ def parseargs(self): if self.testtype == const.BUILDTIME: self.binlogpath = args.binlogpath + if self.testtype == const.IOSINNERLOOP: + self.csprojpath = args.csprojpath + self.editsrcs = args.editsrc.split(';') if args.editsrc else [] + self.editdests = args.editdest.split(';') if args.editdest else [] + self.framework = args.framework + self.configuration = args.configuration + self.msbuildargs = args.msbuildargs or os.environ.get('PERFLAB_MSBUILD_ARGS', '') + # If IOS_RID is set (by .proj PreCommands or setup_helix.py arch + # detection), ensure RuntimeIdentifier in msbuildargs matches it. + ios_rid_env = os.environ.get('IOS_RID', '') + if ios_rid_env and 'RuntimeIdentifier=' in self.msbuildargs: + self.msbuildargs = re.sub( + r'RuntimeIdentifier=\S+', + f'RuntimeIdentifier={ios_rid_env}', + self.msbuildargs) + self.bundleid = args.bundleid + self.deviceid = args.deviceid + self.innerloopiterations = args.innerloopiterations + # Determine device type: explicit arg, or infer from RuntimeIdentifier + # in msbuildargs (ios-arm64 → device, iossimulator-* → simulator) + if args.devicetype: + self.devicetype = args.devicetype + elif 'RuntimeIdentifier=ios-arm64' in self.msbuildargs: + self.devicetype = 'device' + else: + self.devicetype = 'simulator' + if self.testtype == const.ANDROIDINNERLOOP: self.csprojpath = args.csprojpath self.editsrcs = args.editsrc.split(';') if args.editsrc else [] @@ -1320,3 +1414,271 @@ def pre_iteration(): except Exception as e: getLogger().warning("Failed to restore %s: %s", dest, e) androidHelper.close_device(skip_uninstall=True) + elif self.testtype == const.IOSINNERLOOP: + from shutil import copy2, copytree + from performance.common import runninginlab + from performance.constants import UPLOAD_CONTAINER, UPLOAD_STORAGE_URI, UPLOAD_QUEUE + from shared.util import helixuploaddir + import upload + + # --- Validate inputs --- + if not self.csprojpath: + raise Exception("--csproj-path is required for iOS inner loop measurements.") + if not self.bundleid: + raise Exception("--bundle-id is required for iOS inner loop measurements.") + is_physical = (self.devicetype == 'device') + if is_physical and self.deviceid == 'booted': + detected = iOSHelper.detect_connected_device() + if not detected: + raise Exception("Physical device mode requires a device UDID. " + "Set --device-id or IOS_DEVICE_UDID, or connect a device.") + self.deviceid = detected + + # --- Cross-arch RID correction for simulator builds --- + # The .proj defaults RuntimeIdentifier to iossimulator-x64 because + # the original Helix queue (Mac.iPhone.17.Perf) was Intel x64. + # Other queues (e.g. Mac.iPhone.13.Perf) are Apple Silicon, where + # an x64 .app cannot install on an arm64 simulator (mlaunch fails + # with HE0046 "Failed to find matching arch"). Detect the host + # arch here and rewrite the RID in --msbuild-args so the build + # produces a binary matching the simulator we'll deploy to. + # Physical-device builds (ios-arm64) target the iPhone hardware, + # not the host, so we never override their RID. + if not is_physical and self.msbuildargs: + import platform as _platform + host_arch = _platform.machine() + if host_arch == 'arm64' and 'iossimulator-x64' in self.msbuildargs: + self.msbuildargs = self.msbuildargs.replace( + 'iossimulator-x64', 'iossimulator-arm64') + getLogger().info( + "Cross-arch RID correction: host=arm64, rewrote msbuild " + "args to use iossimulator-arm64 (was iossimulator-x64)") + elif host_arch == 'x86_64' and 'iossimulator-arm64' in self.msbuildargs: + self.msbuildargs = self.msbuildargs.replace( + 'iossimulator-arm64', 'iossimulator-x64') + getLogger().info( + "Cross-arch RID correction: host=x86_64, rewrote msbuild " + "args to use iossimulator-x64 (was iossimulator-arm64)") + # Keep IOS_RID env var aligned so versionmanager uses the same + # RID for the linked-DLL lookup below. + m = re.search(r'/p:RuntimeIdentifier=(\S+)', self.msbuildargs) + if m: + os.environ['IOS_RID'] = m.group(1) + + getLogger().info("iOS inner loop: device_type=%s, device_id=%s", self.devicetype, self.deviceid) + scenarioprefix = self.scenarioname or "MAUI iOS Build and Deploy" + + os.makedirs(const.TRACEDIR, exist_ok=True) + # Prefix binlog filenames with runtime flavor to avoid overwrites between runs + runtime_flavor = os.environ.get('RUNTIME_FLAVOR', '') + binlog_prefix = f'{runtime_flavor}-' if runtime_flavor else '' + first_binlog = os.path.join(const.TRACEDIR, f'{binlog_prefix}first-build-and-deploy.binlog') + + # Build base command (no -t:Install for iOS — plain dotnet build) + base_cmd = ['dotnet', 'build', self.csprojpath] + if self.configuration: + base_cmd.extend(['-c', self.configuration]) + if self.framework: + base_cmd.extend(['-f', self.framework]) + if self.msbuildargs: + # _MSBuildArgs in the proj uses both ';' and ' ' as separators. + for arg in re.split(r'[;\s]+', self.msbuildargs): + if arg.strip(): + base_cmd.append(arg.strip()) + + project_dir = os.path.dirname(os.path.abspath(self.csprojpath)) + exename = self.traits.exename + + # --- First build --- + try: + RunCommand(base_cmd + [f'-bl:{first_binlog}'], verbose=True).run() + except CalledProcessError: + getLogger().error("First build failed. Binlog: %s", first_binlog) + raise + + # --- Log SDK and workload versions --- + RunCommand(['dotnet', '--info'], verbose=True).run() + try: + from shared.versionmanager import versions_write_json, versions_write_env, get_sdk_versions + rid = 'ios-arm64' if is_physical else os.environ.get('IOS_RID', 'iossimulator-arm64') + linked_dir = os.path.join(project_dir, 'obj', self.configuration or 'Debug', + self.framework or 'net11.0-ios', rid, 'linked') + if os.path.isdir(linked_dir): + version_dict = get_sdk_versions(linked_dir, False) + versions_file = os.path.join(const.TRACEDIR, f'{binlog_prefix}versions.json') + versions_write_json(version_dict, versions_file) + versions_write_env(version_dict) + getLogger().info("SDK versions: %s", version_dict) + else: + getLogger().warning("Linked DLL dir not found at %s — skipping versions.json", linked_dir) + except Exception as e: + getLogger().warning("Could not extract SDK versions: %s", e) + + # --- Device setup + first deploy --- + iosHelper = iOSHelper() + edit_pairs = [] + try: + app_bundle = iosHelper.find_app_bundle(project_dir, exename, self.configuration, is_physical=is_physical) + first_app_size = _measure_app_size(app_bundle) + getLogger().info("App bundle size: %.2f MB (%d bytes)", first_app_size / 1048576, first_app_size) + iosHelper.setup_device(self.bundleid, app_bundle, self.deviceid, is_physical=is_physical) + iosHelper.sign_app_for_device(app_bundle) + first_install_ms = iosHelper.install_app(app_bundle) + first_startup_ms = iosHelper.measure_cold_startup(self.bundleid) + if first_startup_ms < 0: + raise RuntimeError("First deploy cold startup measurement failed (watchdog event parsing error)") + getLogger().info("First deploy: install=%.1f ms, startup=%d ms", first_install_ms, first_startup_ms) + + # Parse first build binlog + startup = StartupWrapper() + first_build_report = os.path.join(const.TRACEDIR, 'first-build-and-deploy-perf-lab-report.json') + startup.reportjson = first_build_report + saved_upload = self.traits.upload_to_perflab_container + self.traits.add_traits(overwrite=True, apptorun="app", startupmetric=const.IOSINNERLOOP, + tracename=f'{binlog_prefix}first-build-and-deploy.binlog', + scenarioname=scenarioprefix + " - First Build and Deploy", + upload_to_perflab_container=False) + startup.parsetraces(self.traits) + + # Merge first build metrics + install/startup → first E2E report + first_e2e_report = os.path.join(const.TRACEDIR, 'first-debug-e2e-perf-lab-report.json') + _merge_deploy_report(first_build_report, [first_install_ms], [first_startup_ms], first_e2e_report, app_size_bytes=first_app_size) + + # --- Incremental loop --- + if not self.editsrcs or not self.editdests: + raise Exception("--edit-src and --edit-dest are required for incremental builds") + if len(self.editsrcs) != len(self.editdests): + raise Exception("--edit-src and --edit-dest must have the same number of semicolon-separated paths") + + for src, dest in zip(self.editsrcs, self.editdests): + with open(dest, 'r') as f: + original = f.read() + with open(src, 'r') as f: + modified = f.read() + edit_pairs.append((dest, original, modified)) + + num_iterations = self.innerloopiterations + getLogger().info("Starting %d incremental iterations", num_iterations) + + incremental_startup_results = [] + incremental_install_results = [] + incremental_app_size_results = [] + aggregated_counters = {} + report_template = None + + for iteration in range(1, num_iterations + 1): + getLogger().info("=== Incremental iteration %d/%d ===", iteration, num_iterations) + + # Toggle source files (odd → modified, even → original) + for dest, original, modified in edit_pairs: + content = modified if iteration % 2 == 1 else original + with open(dest, 'w') as f: + f.write(content) + + # Build + iter_binlog_name = '%sincremental-build-and-deploy-%d.binlog' % (binlog_prefix, iteration) + iter_binlog = os.path.join(const.TRACEDIR, iter_binlog_name) + try: + RunCommand(base_cmd + [f'-bl:{iter_binlog}'], verbose=True).run() + except CalledProcessError: + getLogger().error("Incremental build %d failed. Binlog: %s", iteration, iter_binlog) + raise + + # Sign (device only — no-op for simulator) + iosHelper.sign_app_for_device(app_bundle) + + # Measure app size after incremental build + iter_app_size = _measure_app_size(app_bundle) + getLogger().info("Iteration %d app bundle size: %.2f MB", iteration, iter_app_size / 1048576) + + # Install + startup + install_ms = iosHelper.install_app(app_bundle) + startup_ms = iosHelper.measure_cold_startup(self.bundleid) + if startup_ms < 0: + raise RuntimeError("Incremental deploy %d cold startup measurement failed (watchdog event parsing error)" % iteration) + getLogger().info("Iteration %d: install=%.1f ms, startup=%d ms", iteration, install_ms, startup_ms) + + incremental_install_results.append(install_ms) + incremental_startup_results.append(startup_ms) + incremental_app_size_results.append(iter_app_size) + + # Parse iteration binlog → temp report + iter_report = os.path.join(const.TRACEDIR, 'incremental-build-report-%d.json' % iteration) + startup.reportjson = iter_report + self.traits.add_traits(overwrite=True, apptorun="app", startupmetric=const.IOSINNERLOOP, + tracename=iter_binlog_name, + scenarioname=scenarioprefix + " - Incremental Build and Deploy", + upload_to_perflab_container=False) + # Clear stale traces upload dir so copytree in parsetraces doesn't collide + helix_upload_dir = helixuploaddir() + if helix_upload_dir is not None: + traces_upload = os.path.join(helix_upload_dir, 'traces') + if os.path.exists(traces_upload): + rmtree(traces_upload) + + startup.parsetraces(self.traits) + + # Extract counters from temp report (may not exist on local runs) + if os.path.exists(iter_report): + with open(iter_report, 'r') as f: + iter_data = json.load(f) + test_obj = iter_data["tests"][0] + if report_template is None: + report_template = {k: v for k, v in test_obj.items() if k != "counters"} + for counter in test_obj["counters"]: + name = counter["name"] + if name not in aggregated_counters: + aggregated_counters[name] = { + "name": name, + "topCounter": counter.get("topCounter", False), + "defaultCounter": counter.get("defaultCounter", False), + "higherIsBetter": counter.get("higherIsBetter", False), + "metricName": counter.get("metricName", "ms"), + "results": [] + } + aggregated_counters[name]["results"].extend(counter.get("results", [])) + os.remove(iter_report) + + # --- Build final incremental report --- + incremental_e2e_report = os.path.join(const.TRACEDIR, 'incremental-debug-e2e-perf-lab-report.json') + final_counters = list(aggregated_counters.values()) + final_counters.append(_make_counter("Install Time", "ms", incremental_install_results)) + final_counters.append(_make_counter("Cold Startup Time", "ms", incremental_startup_results, top=True)) + final_counters.append(_make_counter("App Bundle Size", "bytes", incremental_app_size_results, top=True)) + + final_test = dict(report_template or {}) + final_test["counters"] = final_counters + with open(incremental_e2e_report, 'w') as f: + json.dump({"tests": [final_test]}, f, indent=2) + + # --- Persist reports for local runs --- + results_dir = os.path.join(os.getcwd(), 'results', os.environ.get('RUNTIME_FLAVOR', 'unknown')) + try: + os.makedirs(results_dir, exist_ok=True) + for report in [first_e2e_report, incremental_e2e_report]: + copy2(report, os.path.join(results_dir, os.path.basename(report))) + except Exception as e: + getLogger().warning("Failed to persist reports: %s", e) + + # --- Upload to Helix --- + self.traits.add_traits(overwrite=True, upload_to_perflab_container=saved_upload) + helix_upload_dir = helixuploaddir() + if runninginlab() and helix_upload_dir is not None: + traces_upload = os.path.join(helix_upload_dir, 'traces') + if os.path.exists(traces_upload): + rmtree(traces_upload) + copytree(const.TRACEDIR, traces_upload, dirs_exist_ok=True) + if self.traits.upload_to_perflab_container: + for report_path in [first_e2e_report, incremental_e2e_report]: + upload_code = upload.upload(report_path, UPLOAD_CONTAINER, UPLOAD_QUEUE, UPLOAD_STORAGE_URI) + if upload_code != 0: + sys.exit(upload_code) + + finally: + for dest, original, _modified in edit_pairs: + try: + with open(dest, 'w') as f: + f.write(original) + except Exception as e: + getLogger().warning("Failed to restore %s: %s", dest, e) + iosHelper.cleanup(skip_uninstall=True) diff --git a/src/tools/ScenarioMeasurement/Startup/Startup.cs b/src/tools/ScenarioMeasurement/Startup/Startup.cs index 1e6f6a73bd7..5533330abfd 100644 --- a/src/tools/ScenarioMeasurement/Startup/Startup.cs +++ b/src/tools/ScenarioMeasurement/Startup/Startup.cs @@ -27,6 +27,7 @@ enum MetricType WinUIBlazor, TimeToMain2, BuildTime, + IOSInnerLoop, AndroidInnerLoop, } @@ -292,6 +293,7 @@ static void checkArg(string arg, string name) MetricType.WinUIBlazor => new WinUIBlazorParser(), MetricType.TimeToMain2 => new TimeToMain2Parser(AddTestProcessEnvironmentVariable), MetricType.BuildTime => new BuildTimeParser(), + MetricType.IOSInnerLoop => new IOSInnerLoopParser(), MetricType.AndroidInnerLoop => new AndroidInnerLoopParser(), _ => throw new ArgumentOutOfRangeException(), }; diff --git a/src/tools/ScenarioMeasurement/Util/Parsers/IOSInnerLoopParser.cs b/src/tools/ScenarioMeasurement/Util/Parsers/IOSInnerLoopParser.cs new file mode 100644 index 00000000000..f78f765255c --- /dev/null +++ b/src/tools/ScenarioMeasurement/Util/Parsers/IOSInnerLoopParser.cs @@ -0,0 +1,115 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Microsoft.Build.Logging.StructuredLogger; +using StructuredLogViewer; +using Microsoft.Diagnostics.Tracing; +using Reporting; + +namespace ScenarioMeasurement; + +/// +/// Parses iOS inner loop (build+deploy) target and task durations from a binary log file. +/// +public class IOSInnerLoopParser : IParser +{ + public void EnableKernelProvider(ITraceSession kernel) { throw new NotImplementedException(); } + public void EnableUserProviders(ITraceSession user) { throw new NotImplementedException(); } + + // Task name (case-insensitive) → counter display name + private static readonly Dictionary TrackedTasks = new(StringComparer.OrdinalIgnoreCase) + { + // Shared build tasks + ["Csc"] = "Csc Task Time", + ["XamlCTask"] = "XamlC Task Time", + ["LinkAssembliesNoShrink"] = "LinkAssembliesNoShrink Task Time", + ["FilterAssemblies"] = "FilterAssemblies Task Time", + ["ResolveSdks"] = "ResolveSdks Task Time", + ["ProcessAssemblies"] = "ProcessAssemblies Task Time", + ["GenerateNativeApplicationConfigSources"] = "GenerateNativeApplicationConfigSources Task Time", + // iOS-specific build tasks + ["AOTCompile"] = "AOTCompile Task Time", + ["MonoAOTCompiler"] = "MonoAOTCompiler Task Time", + ["ILLink"] = "ILLink Task Time", + ["CompileNativeCode"] = "CompileNativeCode Task Time", + ["GetFileHash"] = "GetFileHash Task Time", + ["Codesign"] = "Codesign Task Time", + ["CompileNativeFiles"] = "CompileNativeFiles Task Time", + ["LinkNativeCode"] = "LinkNativeCode Task Time", + ["GenerateBundleName"] = "GenerateBundleName Task Time", + ["CreateAssetPack"] = "CreateAssetPack Task Time", + ["ComputeCodesignInputs"] = "ComputeCodesignInputs Task Time", + ["DetectSigningIdentity"] = "DetectSigningIdentity Task Time", + ["CompileAppManifest"] = "CompileAppManifest Task Time", + ["CompileEntitlements"] = "CompileEntitlements Task Time", + ["CreateBindingResourcePackage"] = "CreateBindingResourcePackage Task Time", + ["MTouch"] = "MTouch Task Time", + ["ACTool"] = "ACTool Task Time", + ["IBTool"] = "IBTool Task Time", + ["DSymUtil"] = "DSymUtil Task Time", + }; + + // Target name (case-sensitive, matches MSBuild conventions) → counter display name + private static readonly Dictionary TrackedTargets = new(StringComparer.Ordinal) + { + ["CoreCompile"] = "CoreCompile Target Time", + ["XamlC"] = "XamlC Target Time", + ["_AOTCompile"] = "_AOTCompile Target Time", + ["_CodesignAppBundle"] = "_CodesignAppBundle Target Time", + ["_CompileToNative"] = "_CompileToNative Target Time", + ["_RunILLink"] = "_RunILLink Target Time", + ["_SelectR2RAssemblies"] = "_SelectR2RAssemblies Target Time", + ["_LinkR2RFramework"] = "_LinkR2RFramework Target Time", + ["_CompileNativeExecutable"] = "_CompileNativeExecutable Target Time", + ["_GenerateDSym"] = "_GenerateDSym Target Time", + ["_CreateAppBundle"] = "_CreateAppBundle Target Time", + ["_CopyResourcesToBundle"] = "_CopyResourcesToBundle Target Time", + ["_GenerateBundleName"] = "_GenerateBundleName Target Time", + }; + + public IEnumerable Parse(string binlogFile, string processName, IList pids, string commandLine) + { + if (!File.Exists(binlogFile)) + yield break; + + var build = BinaryLog.ReadBuild(binlogFile); + BuildAnalyzer.AnalyzeBuild(build); + + // Collect task durations + var taskResults = TrackedTasks.ToDictionary(kv => kv.Key, _ => new List(), StringComparer.OrdinalIgnoreCase); + foreach (var task in build.FindChildrenRecursive()) + { + if (taskResults.TryGetValue(task.Name, out var list)) + list.Add(task.Duration.TotalMilliseconds / 1000.0); + } + + // Collect target durations + var targetResults = TrackedTargets.ToDictionary(kv => kv.Key, _ => new List(), StringComparer.Ordinal); + foreach (var target in build.FindChildrenRecursive()) + { + if (targetResults.TryGetValue(target.Name, out var list)) + list.Add(target.Duration.TotalMilliseconds / 1000.0); + } + + // Overall build duration + yield return new Counter { Name = "Build Time", MetricName = "s", DefaultCounter = true, TopCounter = true, + Results = new[] { build.Duration.TotalMilliseconds / 1000.0 } }; + + // Emit task counters + foreach (var (taskName, counterName) in TrackedTasks) + { + var results = taskResults[taskName]; + if (results.Count > 0) + yield return new Counter { Name = counterName, MetricName = "s", DefaultCounter = false, TopCounter = true, Results = results.ToArray() }; + } + + // Emit target counters + foreach (var (targetName, counterName) in TrackedTargets) + { + var results = targetResults[targetName]; + if (results.Count > 0) + yield return new Counter { Name = counterName, MetricName = "s", DefaultCounter = false, TopCounter = true, Results = results.ToArray() }; + } + } +}