Skip to content

Commit 09a4e08

Browse files
stefanosianostefanosiano
committed
Update Android benchmark configurations to include newer API levels and devices
increased number of benchmark runs to increase accuracy increased passing threshold from 5% to 5.5%
1 parent c626414 commit 09a4e08

File tree

4 files changed

+25
-38
lines changed

4 files changed

+25
-38
lines changed

‎.sauce/sentry-uitest-android-benchmark-lite.yml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,13 @@ espresso:
1818

1919
suites:
2020

21-
- name: "Android 11 (api 30)"
21+
- name: "Android 15 Benchmark lite (api 35)"
2222
testOptions:
2323
clearPackageData: true
2424
useTestOrchestrator: true
2525
devices:
26-
- id: Google_Pixel_3a_real # Google Pixel 3a - api 30 (11)
26+
- name: ".*"
27+
platformVersion: "15"
2728

2829
artifacts:
2930
download:

‎.sauce/sentry-uitest-android-benchmark.yml‎

Lines changed: 13 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -19,49 +19,34 @@ espresso:
1919
suites:
2020

2121
# Devices are chosen so that there is a high-end and a low-end device for each api level
22-
- name: "Android 12 (api 31)"
22+
- name: "Android 15 (api 35)"
2323
testOptions:
2424
clearPackageData: true
2525
useTestOrchestrator: true
2626
devices:
27-
- id: Google_Pixel_6_Pro_real_us # Google Pixel 6 Pro - api 31 (12) - high end
28-
- id: Google_Pixel_5_12_real_us # Google Pixel 5 - api 31 (12) - low end
27+
- id: Google_Pixel_9_Pro_XL_15_real_sjc1 # Google Pixel 9 Pro XL - api 35 (15) - high end
28+
- id: Samsung_Galaxy_S23_15_real_sjc1 # Samsung Galaxy S23 - api 35 (15) - mid end
29+
- id: Google_Pixel_6a_15_real_sjc1 # Google Pixel 6a - api 35 (15) - low end
2930

30-
- name: "Android 11 (api 30)"
31+
- name: "Android 14 (api 34)"
3132
testOptions:
3233
clearPackageData: true
3334
useTestOrchestrator: true
3435
devices:
35-
- id: Samsung_Galaxy_S10_Plus_11_real_us # Samsung Galaxy S10+ - api 30 (11) - high end
36-
- id: Google_Pixel_4a_real_us # Google Pixel 4a - api 30 (11) - mid end
37-
- id: Google_Pixel_3a_real # Google Pixel 3a - api 30 (11) - low end
36+
- id: Google_Pixel_9_Pro_XL_real_sjc1 # Google Pixel 9 Pro XL - api 34 (14) - high end
37+
- id: Samsung_Galaxy_A54_real_sjc1 # Samsung Galaxy A54 - api 34 (14) - low end
3838

39-
- name: "Android 10 (api 29)"
39+
- name: "Android 13 (api 33)"
4040
testOptions:
4141
clearPackageData: true
4242
useTestOrchestrator: true
4343
devices:
44-
- id: Google_Pixel_3a_XL_real # Google Pixel 3a XL - api 29 (10)
45-
- id: OnePlus_6T_real # OnePlus 6T - api 29 (10)
44+
- id: Google_Pixel_7_Pro_real_us # Google Pixel 7 Pro - api 33 (13) - high end
45+
- id: Samsung_Galaxy_A32_5G_real_sjc1 # Samsung Galaxy A32 5G - api 33 (13) - low end
4646

47-
# At the time of writing (July, 4, 2022), the market share per android version is:
48-
# 12.0 = 17.54%, 11.0 = 31.65%, 10.0 = 21.92%
49-
# Using these 3 versions we cover 71,11% of all devices out there. Currently, this is enough for benchmarking scope
50-
# Leaving these devices here in case we change mind on them
51-
# devices:
52-
# - id: Samsung_Galaxy_S8_plus_real_us # Samsung Galaxy S8+ - api 28 (9)
53-
# - id: LG_G8_ThinQ_real_us # LG G8 ThinQ - api 28 (9)
54-
# - id: OnePlus_5_real_us # OnePlus 5 - api 27 (8.1.0)
55-
# - id: LG_K30_real_us1 # LG K30 - api 27 (8.1.0)
56-
# - id: HTC_10_real_us # HTC 10 - api 26 (8.0.0)
57-
# - id: Samsung_A3_real # Samsung Galaxy A3 2017 - api 26 (8.0.0)
58-
# - id: ZTE_Axon_7_real2_us # ZTE Axon 7 - api 25 (7.1.1)
59-
# - id: Motorola_Moto_X_Play_real # Motorola Moto X Play - api 25 (7.1.1)
60-
# - id: Samsung_note_5_real_us # Samsung Galaxy Note 5 - api 24 (7.0)
61-
# - id: LG_K10_real # LG K10 - api 24 (7.0)
62-
# - id: Samsung_Galaxy_S6_Edge_Plus_real # Samsung Galaxy S6 Edge+ - api 23 (6.0.1)
63-
# - id: Samsung_Tab_E_real_us # Samsung Tab E - api 23 (6.0.1)
64-
# - id: Amazon_Kindle_Fire_HD_8_real_us # Amazon Kindle Fire HD 8 - api 22 (5.1.1)
47+
# At the time of writing (August, 13, 2025), the market share per android version is:
48+
# 15.0 = 26.75%, 14.0 = 19.5%, 13 = 15.95%
49+
# Using these 3 versions we cover 62.2% of all devices out there. Currently, this is enough for benchmarking scope
6550

6651
artifacts:
6752
download:

‎.sauce/sentry-uitest-android-ui.yml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,29 +18,29 @@ espresso:
1818
testApp: ./sentry-android-integration-tests/sentry-uitest-android/build/outputs/apk/androidTest/release/sentry-uitest-android-release-androidTest.apk
1919
suites:
2020

21-
- name: "Android 14 Ui test (api 34)"
21+
- name: "Android 15 Ui test (api 35)"
2222
testOptions:
2323
clearPackageData: true
2424
useTestOrchestrator: true
2525
devices:
2626
- name: ".*"
27-
platformVersion: "14"
27+
platformVersion: "15"
2828

29-
- name: "Android 13 Ui test (api 33)"
29+
- name: "Android 14 Ui test (api 34)"
3030
testOptions:
3131
clearPackageData: true
3232
useTestOrchestrator: true
3333
devices:
3434
- name: ".*"
35-
platformVersion: "13"
35+
platformVersion: "14"
3636

37-
- name: "Android 11 Ui test (api 31)"
37+
- name: "Android 13 Ui test (api 33)"
3838
testOptions:
3939
clearPackageData: true
4040
useTestOrchestrator: true
4141
devices:
4242
- name: ".*"
43-
platformVersion: "11"
43+
platformVersion: "13"
4444

4545
# Controls what artifacts to fetch when the suite on Sauce Cloud has finished.
4646
artifacts:

‎sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -101,15 +101,16 @@ class SentryBenchmarkTest : BaseBenchmarkTest() {
101101
benchmarkOperationProfiled,
102102
"ProfiledTransaction",
103103
refreshRate,
104+
measuredIterations = 40,
104105
)
105106
comparisonResults.printAllRuns("Profiling Benchmark")
106107
val comparisonResult = comparisonResults.getSummaryResult()
107108
comparisonResult.printResults()
108109

109110
// Currently we just want to assert the cpu overhead
110111
assertTrue(
111-
comparisonResult.cpuTimeIncreasePercentage in 0F..5F,
112-
"Expected ${comparisonResult.cpuTimeIncreasePercentage} to be in range 0 < x < 5",
112+
comparisonResult.cpuTimeIncreasePercentage in 0F..5.5F,
113+
"Expected ${comparisonResult.cpuTimeIncreasePercentage} to be in range 0 < x < 5.5",
113114
)
114115
}
115116

0 commit comments

Comments
 (0)