Skip to content

Commit b81f543

Browse files
pbalcerpbalcer
authored
[CI][Benchmarks] add an option to adjust y-axis per benchmark (#17801)
This is useful for more accurate comparisons between multiple labels.
1 parent 8bbdcde commit b81f543

File tree

5 files changed

+72
-24
lines changed

5 files changed

+72
-24
lines changed

‎devops/scripts/benchmarks/benches/base.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,13 +129,20 @@ def unstable(self) -> str:
129129
def get_tags(self) -> list[str]:
130130
return []
131131

132+
def range(self) -> tuple[float, float]:
133+
return None
134+
132135
def get_metadata(self) -> BenchmarkMetadata:
136+
range = self.range()
137+
133138
return BenchmarkMetadata(
134139
type="benchmark",
135140
description=self.description(),
136141
notes=self.notes(),
137142
unstable=self.unstable(),
138143
tags=self.get_tags(),
144+
range_min=range[0] if range else None,
145+
range_max=range[1] if range else None,
139146
)
140147

141148

‎devops/scripts/benchmarks/benches/compute.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ def additionalMetadata(self) -> dict[str, BenchmarkMetadata]:
9292
"The UR v2 adapter noticeably reduces UR layer overhead, also improving SYCL performance.\n"
9393
"Work is ongoing to reduce the overhead of the SYCL API\n",
9494
tags=["submit", "micro", "SYCL", "UR", "L0"],
95+
range_min=0.0,
9596
),
9697
"SinKernelGraph": BenchmarkMetadata(
9798
type="group",
@@ -315,6 +316,9 @@ def description(self) -> str:
315316
f"Runs 10 simple kernels with minimal execution time to isolate API overhead from kernel execution time. {l0_specific}"
316317
)
317318

319+
def range(self) -> tuple[float, float]:
320+
return (0.0, None)
321+
318322
def bin_args(self) -> list[str]:
319323
return [
320324
f"--Ioq={self.ioq}",

‎devops/scripts/benchmarks/html/index.html

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,10 @@ <h3>Display Options</h3>
4747
<input type="checkbox" id="show-unstable">
4848
Show unstable scenarios
4949
</label>
50+
<label>
51+
<input type="checkbox" id="custom-range">
52+
Adjust Y-axis for comparisons
53+
</label>
5054
</div>
5155
</div>
5256

‎devops/scripts/benchmarks/html/scripts.js

Lines changed: 55 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ function createChart(data, containerId, type) {
9797
];
9898
} else {
9999
return [`${context.dataset.label}:`,
100-
`Value: ${context.parsed.y.toFixed(2)} ${data.unit}`,
100+
`Value: ${context.parsed.y.toFixed(2)} ${data.unit}`,
101101
];
102102
}
103103
}
@@ -111,6 +111,8 @@ function createChart(data, containerId, type) {
111111
text: data.unit
112112
},
113113
grace: '20%',
114+
min: isCustomRangesEnabled() ? data.range_min : null,
115+
max: isCustomRangesEnabled() ? data.range_max : null
114116
}
115117
}
116118
};
@@ -182,7 +184,7 @@ function updateCharts() {
182184
const filterRunData = (chart) => ({
183185
...chart,
184186
runs: Object.fromEntries(
185-
Object.entries(chart.runs).filter(([_, data]) =>
187+
Object.entries(chart.runs).filter(([_, data]) =>
186188
activeRuns.has(data.runName)
187189
)
188190
)
@@ -276,25 +278,25 @@ function createChartContainer(data, canvasId, type) {
276278
// Add tags if present
277279
if (metadata && metadata.tags) {
278280
container.setAttribute('data-tags', metadata.tags.join(','));
279-
281+
280282
// Add tags display
281283
const tagsContainer = document.createElement('div');
282284
tagsContainer.className = 'benchmark-tags';
283-
285+
284286
metadata.tags.forEach(tag => {
285287
const tagElement = document.createElement('span');
286288
tagElement.className = 'tag';
287289
tagElement.textContent = tag;
288290
tagElement.setAttribute('data-tag', tag);
289-
291+
290292
// Add tooltip with tag description
291293
if (benchmarkTags[tag]) {
292294
tagElement.setAttribute('title', benchmarkTags[tag].description);
293295
}
294-
296+
295297
tagsContainer.appendChild(tagElement);
296298
});
297-
299+
298300
container.appendChild(tagsContainer);
299301
}
300302

@@ -493,6 +495,12 @@ function updateURL() {
493495
url.searchParams.set('unstable', 'true');
494496
}
495497

498+
if (!isCustomRangesEnabled()) {
499+
url.searchParams.delete('customRange');
500+
} else {
501+
url.searchParams.set('customRange', 'true');
502+
}
503+
496504
history.replaceState(null, '', url);
497505
}
498506

@@ -505,12 +513,12 @@ function filterCharts() {
505513
const label = container.getAttribute('data-label');
506514
const suite = container.getAttribute('data-suite');
507515
const isUnstable = container.getAttribute('data-unstable') === 'true';
508-
const tags = container.getAttribute('data-tags') ?
509-
container.getAttribute('data-tags').split(',') : [];
516+
const tags = container.getAttribute('data-tags') ?
517+
container.getAttribute('data-tags').split(',') : [];
510518

511519
// Check if benchmark has all active tags (if any are selected)
512-
const hasAllActiveTags = activeTags.size === 0 ||
513-
Array.from(activeTags).every(tag => tags.includes(tag));
520+
const hasAllActiveTags = activeTags.size === 0 ||
521+
Array.from(activeTags).every(tag => tags.includes(tag));
514522

515523
// Hide unstable benchmarks if showUnstable is false
516524
const shouldShow = regex.test(label) &&
@@ -535,16 +543,19 @@ function processTimeseriesData(benchmarkRuns) {
535543

536544
benchmarkRuns.forEach(run => {
537545
run.results.forEach(result => {
546+
const metadata = metadataForLabel(result.label, 'benchmark');
547+
538548
if (!resultsByLabel[result.label]) {
539549
resultsByLabel[result.label] = {
540550
label: result.label,
541551
suite: result.suite,
542552
unit: result.unit,
543553
lower_is_better: result.lower_is_better,
554+
range_min: metadata?.range_min ?? null, // can't use || because js treats 0 as null
555+
range_max: metadata?.range_max ?? null,
544556
runs: {}
545557
};
546558
}
547-
548559
addRunDataPoint(resultsByLabel[result.label], run, result, run.name);
549560
});
550561
});
@@ -568,6 +579,8 @@ function processBarChartsData(benchmarkRuns) {
568579
suite: result.suite,
569580
unit: result.unit,
570581
lower_is_better: result.lower_is_better,
582+
range_min: groupMetadata?.range_min ?? null, // can't use || because js treats 0 as null
583+
range_max: groupMetadata?.range_max ?? null,
571584
labels: [],
572585
datasets: [],
573586
// Add metadata if available
@@ -654,6 +667,8 @@ function processLayerComparisonsData(benchmarkRuns) {
654667
suite: result.suite,
655668
unit: result.unit,
656669
lower_is_better: result.lower_is_better,
670+
range_min: metadata?.range_min ?? null, // can't use || because js treats 0 as null
671+
range_max: metadata?.range_max ?? null,
657672
runs: {},
658673
benchmarkLabels: [],
659674
description: metadata?.description || null,
@@ -760,26 +775,37 @@ function isUnstableEnabled() {
760775
return unstableToggle.checked;
761776
}
762777

778+
function isCustomRangesEnabled() {
779+
const rangesToggle = document.getElementById('custom-range');
780+
return rangesToggle.checked;
781+
}
782+
763783
function setupToggles() {
764784
const notesToggle = document.getElementById('show-notes');
765785
const unstableToggle = document.getElementById('show-unstable');
786+
const customRangeToggle = document.getElementById('custom-range');
766787

767-
notesToggle.addEventListener('change', function() {
788+
notesToggle.addEventListener('change', function () {
768789
// Update all note elements visibility
769790
document.querySelectorAll('.benchmark-note').forEach(note => {
770791
note.style.display = isNotesEnabled() ? 'block' : 'none';
771792
});
772793
updateURL();
773794
});
774795

775-
unstableToggle.addEventListener('change', function() {
796+
unstableToggle.addEventListener('change', function () {
776797
// Update all unstable warning elements visibility
777798
document.querySelectorAll('.benchmark-unstable').forEach(warning => {
778799
warning.style.display = isUnstableEnabled() ? 'block' : 'none';
779800
});
780801
filterCharts();
781802
});
782803

804+
customRangeToggle.addEventListener('change', function () {
805+
// redraw all charts
806+
updateCharts();
807+
});
808+
783809
// Initialize from URL params if present
784810
const notesParam = getQueryParam('notes');
785811
const unstableParam = getQueryParam('unstable');
@@ -793,13 +819,18 @@ function setupToggles() {
793819
let showUnstable = unstableParam === 'true';
794820
unstableToggle.checked = showUnstable;
795821
}
822+
823+
const customRangesParam = getQueryParam('customRange');
824+
if (customRangesParam !== null) {
825+
customRangeToggle.checked = customRangesParam === 'true';
826+
}
796827
}
797828

798829
function setupTagFilters() {
799830
tagFiltersContainer = document.getElementById('tag-filters');
800831

801832
const allTags = [];
802-
833+
803834
if (benchmarkTags) {
804835
for (const tag in benchmarkTags) {
805836
if (!allTags.includes(tag)) {
@@ -812,17 +843,17 @@ function setupTagFilters() {
812843
allTags.forEach(tag => {
813844
const tagContainer = document.createElement('div');
814845
tagContainer.className = 'tag-filter';
815-
846+
816847
const checkbox = document.createElement('input');
817848
checkbox.type = 'checkbox';
818849
checkbox.id = `tag-${tag}`;
819850
checkbox.className = 'tag-checkbox';
820851
checkbox.dataset.tag = tag;
821-
852+
822853
const label = document.createElement('label');
823854
label.htmlFor = `tag-${tag}`;
824855
label.textContent = tag;
825-
856+
826857
// Add info icon with tooltip if tag description exists
827858
if (benchmarkTags[tag]) {
828859
const infoIcon = document.createElement('span');
@@ -831,16 +862,16 @@ function setupTagFilters() {
831862
infoIcon.title = benchmarkTags[tag].description;
832863
label.appendChild(infoIcon);
833864
}
834-
835-
checkbox.addEventListener('change', function() {
865+
866+
checkbox.addEventListener('change', function () {
836867
if (this.checked) {
837868
activeTags.add(tag);
838869
} else {
839870
activeTags.delete(tag);
840871
}
841872
filterCharts();
842873
});
843-
874+
844875
tagContainer.appendChild(checkbox);
845876
tagContainer.appendChild(label);
846877
tagFiltersContainer.appendChild(tagContainer);
@@ -849,18 +880,18 @@ function setupTagFilters() {
849880

850881
function toggleAllTags(select) {
851882
const checkboxes = document.querySelectorAll('.tag-checkbox');
852-
883+
853884
checkboxes.forEach(checkbox => {
854885
checkbox.checked = select;
855886
const tag = checkbox.dataset.tag;
856-
887+
857888
if (select) {
858889
activeTags.add(tag);
859890
} else {
860891
activeTags.delete(tag);
861892
}
862893
});
863-
894+
864895
filterCharts();
865896
}
866897

‎devops/scripts/benchmarks/utils/result.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,8 @@ class BenchmarkMetadata:
5959
notes: str = None
6060
unstable: str = None
6161
tags: list[str] = field(default_factory=list)
62+
range_min: float = None
63+
range_max: float = None
6264

6365

6466
@dataclass_json

0 commit comments

Comments
 (0)