Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add sycl-bench benchmarks #2047

Closed
wants to merge 28 commits into from
Closed
Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
939b4d0
add syclbench.py
mateuszpn Sep 3, 2024
335faaa
unisa-hpc/sycl-bench benchmarks added
mateuszpn Sep 4, 2024
56cf9cf
Merge branch 'main' into add-sycl-bench
mateuszpn Sep 4, 2024
d5b553a
Merge branch 'oneapi-src:main' into add-sycl-bench
mateuszpn Sep 4, 2024
5f374f6
Benchamrk names simplified
mateuszpn Sep 5, 2024
c4854bf
Merge branch 'add-sycl-bench' of github.com:mateuszpn/unified-runtime…
mateuszpn Sep 5, 2024
0ab5039
Merge branch 'oneapi-src:main' into add-sycl-bench
mateuszpn Sep 6, 2024
535c723
update
mateuszpn Sep 10, 2024
7d751ba
update
mateuszpn Sep 10, 2024
6895f21
update
mateuszpn Sep 10, 2024
ff3a0f1
WIP
mateuszpn Sep 12, 2024
20d6859
Performance chart and geometrical mean
mateuszpn Sep 12, 2024
162681b
Merge branch 'oneapi-src:main' into add-sycl-bench
mateuszpn Sep 16, 2024
2af01f7
push benchmarks results
mateuszpn Sep 16, 2024
98f338b
Merge branch 'add-sycl-bench' of github.com:mateuszpn/unified-runtime…
mateuszpn Sep 16, 2024
b1acce4
WIP [skip ci]
mateuszpn Sep 17, 2024
1499fdd
WIP [skip ci]
mateuszpn Sep 17, 2024
8479e59
Merge branch 'oneapi-src:main' into add-sycl-bench
mateuszpn Sep 17, 2024
3b2f85e
png added
mateuszpn Sep 17, 2024
8df45d8
Merge branch 'add-sycl-bench' of github.com:mateuszpn/unified-runtime…
mateuszpn Sep 17, 2024
716f103
action added [skip ci]
mateuszpn Sep 17, 2024
50309a8
ascii bar chart added to benchmarks results [skip ci]
mateuszpn Sep 19, 2024
a2affaa
ascii bar fix [skip ci]
mateuszpn Sep 19, 2024
0358db2
bar chart update [skip ci]
mateuszpn Sep 20, 2024
90608c8
update [skip ci]
mateuszpn Sep 20, 2024
ebe4160
update [skip ci]
mateuszpn Sep 20, 2024
379e1bd
update [skip ci]
mateuszpn Sep 20, 2024
7af46ef
Merge branch 'oneapi-src:main' into add-sycl-bench
mateuszpn Sep 20, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion scripts/benchmarks/benches/compute.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ def run(self, env_vars) -> Result:

result = self.run_bench(command, env_vars)
(label, mean) = self.parse_output(result)
return Result(label=label, value=mean, command=command, env=env_vars, stdout=result, lower_is_better=self.lower_is_better())
return [ Result(label=label, value=mean, command=command, env=env_vars, stdout=result, lower_is_better=self.lower_is_better()) ]

def parse_output(self, output):
csv_file = io.StringIO(output)
Expand Down
369 changes: 369 additions & 0 deletions scripts/benchmarks/benches/syclbench.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,369 @@
# Copyright (C) 2024 Intel Corporation
# Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
# See LICENSE.TXT
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

import os
import csv
import io
from utils.utils import run, git_clone, create_build_path
from .base import Benchmark
from .result import Result
from .options import options

class SyclBench:
def __init__(self, directory):
self.directory = directory
self.built = False
self.setup()
return

def setup(self):
if self.built:
return

repo_path = git_clone(self.directory, "sycl-bench-repo", "https://github.com/mateuszpn/sycl-bench.git", "1e6ab2cfd004a72c5336c26945965017e06eab71")
build_path = create_build_path(self.directory, 'sycl-bench-build')

configure_command = [
"cmake",
f"-B {build_path}",
f"-S {repo_path}",
f"-DCMAKE_BUILD_TYPE=Release",
f"-DCMAKE_CXX_COMPILER={options.sycl}/bin/clang++",
f"-DCMAKE_C_COMPILER={options.sycl}/bin/clang",
f"-DSYCL_IMPL=dpcpp"
]
run(configure_command, add_sycl=True)

run(f"cmake --build {build_path} -j", add_sycl=True)

self.built = True
self.bins = build_path
return

class SyclBenchmark(Benchmark):
def __init__(self, bench, name, test):
self.bench = bench
self.bench_name = name
self.test = test
super().__init__(bench.directory)

def bin_args(self) -> list[str]:
return []

def extra_env_vars(self) -> dict:
return {}

def unit(self):
return "ms"

def setup(self):
self.bench.setup()
self.benchmark_bin = os.path.join(self.bench.bins, self.bench_name)

def run(self, env_vars) -> Result:
outputfile = f"{self.bench.directory}/{self.test}.csv"
command = [
f"{self.benchmark_bin}",
f"--warmup-run",
f"--num-runs=3",
f"--output={outputfile}"
]

command += self.bin_args()
env_vars.update(self.extra_env_vars())

result = self.run_bench(command, env_vars)

with open(outputfile, 'r') as f:
reader = csv.reader(f)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure if I follow. We open an "output" file that contains some benchmark definitions? Shouldn't that be input file then?

Copy link
Contributor Author

@mateuszpn mateuszpn Sep 26, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is an output file containing results - when put in file, they are structured as csv and easy to analyze. I must analyze the output, whether it is in stdout or file, to return results.

res_list = []
for row in reader:
if not row[0].startswith('#'):
res_list.append(
Result(label=row[0],
value=float(row[12]) * 1000, # convert to ms
command=command,
env=env_vars,
stdout=result))

median_list = []
for label in set(result.label for result in res_list):
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is already done for all the benchmarks in main.py. why repeat it?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Removed

values = [result.value for result in res_list if result.label == label]
median_value = sorted(values)[len(values) // 2]
median_list.append(Result(label=label, value=median_value, command=command, env=env_vars, stdout=result))

return median_list

def teardown(self):
return

def name(self):
return self.test

class Arith(SyclBenchmark):
def __init__(self, bench):
super().__init__(bench, "arith", "Arith_int32_512")

def bin_args(self) -> list[str]:
return [
f"--size=16384",
]

class TwoDConvolution(SyclBenchmark):
def __init__(self, bench):
super().__init__(bench, "2DConvolution", "2DConvolution")

class Two_mm(SyclBenchmark):
def __init__(self, bench):
super().__init__(bench, "2mm", "2mm")

def bin_args(self) -> list[str]:
return [
f"--size=512",
]

class Three_mm(SyclBenchmark):
def __init__(self, bench):
super().__init__(bench, "3mm", "3mm")

def bin_args(self) -> list[str]:
return [
f"--size=512",
]

class Atax(SyclBenchmark):
def __init__(self, bench):
super().__init__(bench, "atax", "Atax")

def bin_args(self) -> list[str]:
return [
f"--size=8192",
]

class Atomic_reduction(SyclBenchmark):
def __init__(self, bench):
super().__init__(bench, "atomic_reduction", "ReductionAtomic_fp64")

class Bicg(SyclBenchmark):
def __init__(self, bench):
super().__init__(bench, "bicg", "Bicg")

def bin_args(self) -> list[str]:
return [
f"--size=8192",
]

class Correlation(SyclBenchmark):
def __init__(self, bench):
super().__init__(bench, "correlation", "Correlation")

def bin_args(self) -> list[str]:
return [
f"--size=512",
]

class Covariance(SyclBenchmark):
def __init__(self, bench):
super().__init__(bench, "covariance", "Covariance")

def bin_args(self) -> list[str]:
return [
f"--size=512",
]

class Gemm(SyclBenchmark):
def __init__(self, bench):
super().__init__(bench, "gemm", "Gemm")

def bin_args(self) -> list[str]:
return [
f"--size=1024",
]

class Gesumv(SyclBenchmark):
def __init__(self, bench):
super().__init__(bench, "gesummv", "Gesummv")

def bin_args(self) -> list[str]:
return [
f"--size=8192",
]

class Gramschmidt(SyclBenchmark):
def __init__(self, bench):
super().__init__(bench, "gramschmidt", "Gramschmidt")

def bin_args(self) -> list[str]:
return [
f"--size=512",
]

class KMeans(SyclBenchmark):
def __init__(self, bench):
super().__init__(bench, "kmeans", "Kmeans")

def bin_args(self) -> list[str]:
return [
f"--size=67108864",
]

class LinRegCoeff(SyclBenchmark):
def __init__(self, bench):
super().__init__(bench, "lin_reg_coeff", "LinearRegressionCoeff")

class LinRegError(SyclBenchmark):
def __init__(self, bench):
super().__init__(bench, "lin_reg_error", "LinearRegression")

class MatmulChain(SyclBenchmark):
def __init__(self, bench):
super().__init__(bench, "matmulchain", "MatmulChain")

def bin_args(self) -> list[str]:
return [
f"--size=1024",
]

# ** bad input file path **
#
# class Median(SyclBenchmark):
# def __init__(self, bench):
# super().__init__(bench, "median", "MedianFilter")
#
# def bin_args(self) -> list[str]:
# return [
# f"--size=512",
# ]

class MolDyn(SyclBenchmark):
def __init__(self, bench):
super().__init__(bench, "mol_dyn", "MolecularDynamics")


class Mvt(SyclBenchmark):
def __init__(self, bench):
super().__init__(bench, "mvt", "Mvt")

def bin_args(self) -> list[str]:
return [
f"--size=16384",
]

# ** verification fail **
#
# class NBody(SyclBenchmark):
# def __init__(self, bench):
# super().__init__(bench, "nbody", "NBody_")

class Sf(SyclBenchmark):
def __init__(self, bench):
super().__init__(bench, "sf", "sf_16")

def bin_args(self) -> list[str]:
return [
f"--size=--size=100000000",
]

# bad input file path
#
# class SobelX(SyclBenchmark):
# def __init__(self, bench):
# super().__init__(bench, "sobel", "SobelFilter")

class Syr2k(SyclBenchmark):
def __init__(self, bench):
super().__init__(bench, "syr2k", "Syr2k")

def bin_args(self) -> list[str]:
return [
f"--size=1024",
]

class Syrk(SyclBenchmark):
def __init__(self, bench):
super().__init__(bench, "syrk", "Syrk")

def bin_args(self) -> list[str]:
return [
f"--size=1024",
]

# multi benchmarks
class Blocked_transform(SyclBenchmark):
def __init__(self, bench):
super().__init__(bench, "blocked_transform", "BlockedTransform_multi")

def bin_args(self) -> list[str]:
return [
f"--size=512",
]

class DagTaskI(SyclBenchmark):
def __init__(self, bench):
super().__init__(bench, "dag_task_throughput_independent", "IndependentDAGTaskThroughput_multi")

def bin_args(self) -> list[str]:
return [
f"--size=512",
]

class DagTaskS(SyclBenchmark):
def __init__(self, bench):
super().__init__(bench, "dag_task_throughput_sequential", "DAGTaskThroughput_multi")

def bin_args(self) -> list[str]:
return [
f"--size=512",
]

class HostDevBandwidth(SyclBenchmark):
def __init__(self, bench):
super().__init__(bench, "host_device_bandwidth", "HostDeviceBandwidth_multi")

class LocalMem(SyclBenchmark):
def __init__(self, bench):
super().__init__(bench, "local_mem", f"LocalMem_multi")

def bin_args(self) -> list[str]:
return [
f"--size=512",
]

class Pattern_L2(SyclBenchmark):
def __init__(self, bench):
super().__init__(bench, "pattern_L2", "L2_multi")

class Reduction(SyclBenchmark):
def __init__(self, bench):
super().__init__(bench, "reduction", "Pattern_Reduction_multi")

class ScalarProd(SyclBenchmark):
def __init__(self, bench):
super().__init__(bench, "scalar_prod", "ScalarProduct_multi")

class SegmentReduction(SyclBenchmark):
def __init__(self, bench):
super().__init__(bench, "segmentedreduction", "Pattern_SegmentedReduction_multi")

class UsmAccLatency(SyclBenchmark):
def __init__(self, bench):
super().__init__(bench, "usm_accessors_latency", "USM_Latency_multi")

class UsmAllocLatency(SyclBenchmark):
def __init__(self, bench):
super().__init__(bench, "usm_allocation_latency", "USM_Allocation_latency_multi")

class UsmInstrMix(SyclBenchmark):
def __init__(self, bench):
super().__init__(bench, "usm_instr_mix", "USM_Instr_Mix_multi")

class UsmPinnedOverhead(SyclBenchmark):
def __init__(self, bench):
super().__init__(bench, "usm_pinned_overhead", "USM_Pinned_Overhead_multi")

class VecAdd(SyclBenchmark):
def __init__(self, bench):
super().__init__(bench, "vec_add", "VectorAddition_multi")

2 changes: 1 addition & 1 deletion scripts/benchmarks/benches/velocity.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ def run(self, env_vars) -> Result:

result = self.run_bench(command, env_vars)

return Result(label=self.bench_name, value=self.parse_output(result), command=command, env=env_vars, stdout=result, lower_is_better=self.lower_is_better())
return [ Result(label=self.bench_name, value=self.parse_output(result), command=command, env=env_vars, stdout=result, lower_is_better=self.lower_is_better()) ]

def teardown(self):
return
Loading