diff --git a/eng/performance/maui_scenarios_android.proj b/eng/performance/maui_scenarios_android.proj index 07440d846b5..3e50a517e7d 100644 --- a/eng/performance/maui_scenarios_android.proj +++ b/eng/performance/maui_scenarios_android.proj @@ -71,7 +71,7 @@ - $(Python) pre.py publish -f $(PERFLAB_Framework)-android -r android-arm64 --self-contained --msbuild="$(_MSBuildArgs)" --binlog $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName)\%(PreparePayloadWorkItem.ScenarioDirectoryName).$(RunConfigsString).binlog -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName) + $(Python) pre.py publish -f $(PERFLAB_Framework)-android -r android-arm64 --self-contained -c $(BuildConfig) --msbuild="$(_MSBuildArgs)" --binlog $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName)\%(PreparePayloadWorkItem.ScenarioDirectoryName).$(RunConfigsString).binlog -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName) %(PreparePayloadWorkItem.PayloadDirectory) diff --git a/eng/performance/maui_scenarios_ios.proj b/eng/performance/maui_scenarios_ios.proj index 3b8e83b1fab..c5041b8985b 100644 --- a/eng/performance/maui_scenarios_ios.proj +++ b/eng/performance/maui_scenarios_ios.proj @@ -58,7 +58,7 @@ - sudo xcode-select -s /Applications/Xcode_26.1.1.app; $(Python) pre.py publish -f $(PERFLAB_Framework)-ios --self-contained -c Release -r ios-arm64 --msbuild="$(_MSBuildArgs)" --binlog $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName)/%(PreparePayloadWorkItem.ScenarioDirectoryName).binlog -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName); cd ../; zip -r %(PreparePayloadWorkItem.ScenarioDirectoryName).zip %(PreparePayloadWorkItem.ScenarioDirectoryName) + sudo xcode-select -s /Applications/Xcode_26.2.app; $(Python) pre.py publish -f $(PERFLAB_Framework)-ios --self-contained -c $(BuildConfig) -r ios-arm64 --msbuild="$(_MSBuildArgs)" --binlog $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName)/%(PreparePayloadWorkItem.ScenarioDirectoryName).binlog -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName); cd ../; zip -r %(PreparePayloadWorkItem.ScenarioDirectoryName).zip %(PreparePayloadWorkItem.ScenarioDirectoryName) %(PreparePayloadWorkItem.PayloadDirectory) diff --git a/eng/pipelines/sdk-perf-jobs.yml b/eng/pipelines/sdk-perf-jobs.yml index bc709cb7b2d..245737c12f4 100644 --- a/eng/pipelines/sdk-perf-jobs.yml +++ b/eng/pipelines/sdk-perf-jobs.yml @@ -543,6 +543,164 @@ jobs: ${{ each parameter in parameters.jobParameters }}: ${{ parameter.key }}: ${{ parameter.value }} + # Maui Android Debug scenario benchmarks (Mono ProfiledAOT) + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-android-arm64-pixel + - win-x64-android-arm64-galaxy + isPublic: false + jobParameters: + runKind: maui_scenarios_android + projectFileName: maui_scenarios_android.proj + channels: + - main + runtimeFlavor: mono + codeGenType: ProfiledAOT + buildConfig: Debug + additionalJobIdentifier: Mono_Debug + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Maui Android Debug scenario benchmarks (Mono AOT) + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-android-arm64-pixel + - win-x64-android-arm64-galaxy + isPublic: false + jobParameters: + runKind: maui_scenarios_android + projectFileName: maui_scenarios_android.proj + channels: + - main + runtimeFlavor: mono + codeGenType: AOT + buildConfig: Debug + additionalJobIdentifier: Mono_Debug + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Maui Android Debug scenario benchmarks (CoreCLR JIT) + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-android-arm64-pixel + - win-x64-android-arm64-galaxy + isPublic: false + jobParameters: + runKind: maui_scenarios_android + projectFileName: maui_scenarios_android.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: JIT + buildConfig: Debug + additionalJobIdentifier: CoreCLR_Debug + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Maui Android Debug scenario benchmarks (CoreCLR R2R) + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-android-arm64-pixel + - win-x64-android-arm64-galaxy + isPublic: false + jobParameters: + runKind: maui_scenarios_android + projectFileName: maui_scenarios_android.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: R2R + buildConfig: Debug + additionalJobIdentifier: CoreCLR_Debug + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Maui Android Debug scenario benchmarks (CoreCLR R2R Composite) + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-android-arm64-pixel + - win-x64-android-arm64-galaxy + isPublic: false + jobParameters: + runKind: maui_scenarios_android + projectFileName: maui_scenarios_android.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: R2RComposite + buildConfig: Debug + additionalJobIdentifier: CoreCLR_Debug + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Maui Android Debug scenario benchmarks (CoreCLR NativeAOT) + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-android-arm64-pixel + - win-x64-android-arm64-galaxy + isPublic: false + jobParameters: + runKind: maui_scenarios_android + projectFileName: maui_scenarios_android.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: NativeAOT + buildConfig: Debug + additionalJobIdentifier: CoreCLR_Debug + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Maui iOS Debug Mono scenario benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - osx-x64-ios-arm64 + isPublic: false + jobParameters: + runKind: maui_scenarios_ios + projectFileName: maui_scenarios_ios.proj + channels: + - main + runtimeFlavor: mono + codeGenType: FullAOT + buildConfig: Debug + additionalJobIdentifier: Mono_Debug + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Maui iOS Debug Native AOT scenario benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - osx-x64-ios-arm64 + isPublic: false + jobParameters: + runKind: maui_scenarios_ios + projectFileName: maui_scenarios_ios.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: NativeAOT + buildConfig: Debug + additionalJobIdentifier: CoreCLR_Debug + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + # Maui scenario benchmarks - ${{ if false }}: - template: /eng/pipelines/templates/build-machine-matrix.yml @@ -848,4 +1006,4 @@ jobs: - 9.0 - 8.0 ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/scripts/run_performance_job.py b/scripts/run_performance_job.py index d98116c3ba9..c27fe53d120 100644 --- a/scripts/run_performance_job.py +++ b/scripts/run_performance_job.py @@ -20,6 +20,8 @@ from performance.logger import setup_loggers from send_to_helix import PerfSendToHelixArgs, perf_send_to_helix +DEFAULT_BUILD_CONFIG = "Release" + def output_counters_for_crank(reports: list[Any]): print("#StartJobStatistics") @@ -113,7 +115,7 @@ class RunPerformanceJobArgs: os_version: Optional[str] = None dotnet_version_link: Optional[str] = None target_csproj: Optional[str] = None - build_config: str = "Release" + build_config: str = DEFAULT_BUILD_CONFIG live_libraries_build_config: Optional[str] = None cross_build: bool = False @@ -123,6 +125,7 @@ def get_pre_commands( internal: bool, runtime_type: str, codegen_type: str, + build_config: str, v8_version: str): helix_pre_commands: list[str] = [] @@ -296,6 +299,16 @@ def get_pre_commands( helix_pre_commands += ["%HELIX_CORRELATION_PAYLOAD%\\monoaot\\mono-aot-cross --llvm --version"] else: helix_pre_commands += ["$HELIX_CORRELATION_PAYLOAD/monoaot/mono-aot-cross --llvm --version"] + + # If we are running not release, make sure we make that very clear. + if build_config.lower() != "release": + banner = [ + "(echo ======================================================)", + f"(echo NON-RELEASE BUILD CONFIG: {build_config})", + "(echo ======================================================)", + ] + + helix_pre_commands = banner + helix_pre_commands return helix_pre_commands @@ -475,7 +488,8 @@ def get_run_configurations( runtime_flavor: Optional[str] = None, ios_llvm_build: bool = False, ios_strip_symbols: bool = False, - javascript_engine: Optional[str] = None): + javascript_engine: Optional[str] = None, + build_config: Optional[str] = None): configurations = { "CompilationMode": "Tiered", "RunKind": run_kind } @@ -531,6 +545,8 @@ def get_run_configurations( raise Exception("Runtime flavor must be specified for maui_scenarios_android") configurations["CodegenType"] = str(codegen_type) configurations["RuntimeType"] = str(runtime_flavor) + if build_config is not None and build_config != DEFAULT_BUILD_CONFIG: + configurations["BuildConfig"] = build_config # .NET iOS and .NET MAUI iOS sample app scenarios if run_kind == "maui_scenarios_ios": @@ -538,6 +554,8 @@ def get_run_configurations( raise Exception("Runtime flavor must be specified for maui_scenarios_ios") configurations["CodegenType"] = str(codegen_type) configurations["RuntimeType"] = str(runtime_flavor) + if build_config is not None and build_config != DEFAULT_BUILD_CONFIG: + configurations["BuildConfig"] = build_config return configurations @@ -697,7 +715,8 @@ def run_performance_job(args: RunPerformanceJobArgs): configurations = get_run_configurations( args.run_kind, args.runtime_type, args.codegen_type, args.pgo_run_type, args.physical_promotion_run_type, args.r2r_run_type, args.experiment_name, args.linking_type, - args.runtime_flavor, args.ios_llvm_build, args.ios_strip_symbols, args.javascript_engine + args.runtime_flavor, args.ios_llvm_build, args.ios_strip_symbols, args.javascript_engine, + args.build_config ) ci_setup_arguments = ci_setup.CiSetupArgs( @@ -925,7 +944,7 @@ def run_performance_job(args: RunPerformanceJobArgs): else: agent_python = "python3" - helix_pre_commands = get_pre_commands(args.os_group, args.os_distro, args.internal, args.runtime_type, args.codegen_type, v8_version) + helix_pre_commands = get_pre_commands(args.os_group, args.os_distro, args.internal, args.runtime_type, args.codegen_type, args.build_config, v8_version) helix_post_commands = get_post_commands(args.os_group, args.internal, args.runtime_type) ci_setup_arguments.local_build = args.local_build @@ -1048,6 +1067,7 @@ def publish_dotnet_app_to_payload(payload_dir_name: str, csproj_path: str, self_ os.environ["Python"] = agent_python os.environ["RuntimeFlavor"] = args.runtime_flavor or '' os.environ["CodegenType"] = args.codegen_type or '' + os.environ["BuildConfig"] = args.build_config or DEFAULT_BUILD_CONFIG # TODO: See if these commands are needed for linux as they were being called before but were failing. if args.os_group == "windows" or args.os_group == "osx":