Skip to content

Commit df013c5

Browse files
martin-frbgmartin-frbg
authored
Merge pull request #5358 from iha-taisei/dot_unroll
Performance improvements of [SD]DOT with loop-unrolling on A64FX
2 parents 36c2589 + f7ad906 commit df013c5

File tree

4 files changed

+187
-0
lines changed

4 files changed

+187
-0
lines changed

‎kernel/arm64/KERNEL.A64FX

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,6 @@ SGEMVNKERNEL = gemv_n_sve_v4x3.c
44
DGEMVNKERNEL = gemv_n_sve_v4x3.c
55
SGEMVTKERNEL = gemv_t_sve_v4x3.c
66
DGEMVTKERNEL = gemv_t_sve_v4x3.c
7+
8+
DDOTKERNEL = dot_sve_v8.c
9+
SDOTKERNEL = dot_sve_v8.c

‎kernel/arm64/dot.c

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
4040
#endif
4141

4242
#ifdef USE_SVE
43+
#ifdef DOT_KERNEL_SVE
44+
#include DOT_KERNEL_SVE
45+
#else
4346
#include "dot_kernel_sve.c"
4447
#endif
48+
#endif
4549
#include "dot_kernel_asimd.c"
4650

4751
#if defined(SMP)

‎kernel/arm64/dot_kernel_sve_v8.c

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
/***************************************************************************
2+
Copyright (c) 2025, The OpenBLAS Project
3+
All rights reserved.
4+
5+
Redistribution and use in source and binary forms, with or without
6+
modification, are permitted provided that the following conditions are
7+
met:
8+
9+
1. Redistributions of source code must retain the above copyright
10+
notice, this list of conditions and the following disclaimer.
11+
12+
2. Redistributions in binary form must reproduce the above copyright
13+
notice, this list of conditions and the following disclaimer in
14+
the documentation and/or other materials provided with the
15+
distribution.
16+
3. Neither the name of the OpenBLAS project nor the names of
17+
its contributors may be used to endorse or promote products
18+
derived from this software without specific prior written
19+
permission.
20+
21+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
22+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24+
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
25+
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
27+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
28+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
29+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
30+
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31+
*****************************************************************************/
32+
33+
#include <arm_sve.h>
34+
#include "common.h"
35+
36+
#ifdef DOUBLE
37+
#define SV_COUNT svcntd
38+
#define SV_TYPE svfloat64_t
39+
#define SV_TRUE svptrue_b64
40+
#define SV_WHILE svwhilelt_b64_s64
41+
#define SV_DUP svdup_f64
42+
#else
43+
#define SV_COUNT svcntw
44+
#define SV_TYPE svfloat32_t
45+
#define SV_TRUE svptrue_b32
46+
#define SV_WHILE svwhilelt_b32_s64
47+
#define SV_DUP svdup_f32
48+
#endif
49+
50+
static FLOAT dot_kernel_sve(BLASLONG n, FLOAT* x, FLOAT* y)
51+
{
52+
SV_TYPE temp0_vec = SV_DUP(0.0);
53+
SV_TYPE temp1_vec = SV_DUP(0.0);
54+
SV_TYPE temp2_vec = SV_DUP(0.0);
55+
SV_TYPE temp3_vec = SV_DUP(0.0);
56+
SV_TYPE temp4_vec = SV_DUP(0.0);
57+
SV_TYPE temp5_vec = SV_DUP(0.0);
58+
SV_TYPE temp6_vec = SV_DUP(0.0);
59+
SV_TYPE temp7_vec = SV_DUP(0.0);
60+
61+
BLASLONG i = 0;
62+
BLASLONG sve_size = SV_COUNT();
63+
64+
while ((i + sve_size * 8 - 1) < n) {
65+
FLOAT *x0_ptr = x + i;
66+
SV_TYPE x0_vec = svld1_vnum(SV_TRUE(), x0_ptr, 0);
67+
SV_TYPE x1_vec = svld1_vnum(SV_TRUE(), x0_ptr, 1);
68+
SV_TYPE x2_vec = svld1_vnum(SV_TRUE(), x0_ptr, 2);
69+
SV_TYPE x3_vec = svld1_vnum(SV_TRUE(), x0_ptr, 3);
70+
SV_TYPE x4_vec = svld1_vnum(SV_TRUE(), x0_ptr, 4);
71+
SV_TYPE x5_vec = svld1_vnum(SV_TRUE(), x0_ptr, 5);
72+
SV_TYPE x6_vec = svld1_vnum(SV_TRUE(), x0_ptr, 6);
73+
SV_TYPE x7_vec = svld1_vnum(SV_TRUE(), x0_ptr, 7);
74+
75+
FLOAT *y0_ptr = y + i;
76+
SV_TYPE y0_vec = svld1_vnum(SV_TRUE(), y0_ptr, 0);
77+
SV_TYPE y1_vec = svld1_vnum(SV_TRUE(), y0_ptr, 1);
78+
SV_TYPE y2_vec = svld1_vnum(SV_TRUE(), y0_ptr, 2);
79+
SV_TYPE y3_vec = svld1_vnum(SV_TRUE(), y0_ptr, 3);
80+
SV_TYPE y4_vec = svld1_vnum(SV_TRUE(), y0_ptr, 4);
81+
SV_TYPE y5_vec = svld1_vnum(SV_TRUE(), y0_ptr, 5);
82+
SV_TYPE y6_vec = svld1_vnum(SV_TRUE(), y0_ptr, 6);
83+
SV_TYPE y7_vec = svld1_vnum(SV_TRUE(), y0_ptr, 7);
84+
85+
temp0_vec = svmla_x(SV_TRUE(), temp0_vec, x0_vec, y0_vec);
86+
temp1_vec = svmla_x(SV_TRUE(), temp1_vec, x1_vec, y1_vec);
87+
temp2_vec = svmla_x(SV_TRUE(), temp2_vec, x2_vec, y2_vec);
88+
temp3_vec = svmla_x(SV_TRUE(), temp3_vec, x3_vec, y3_vec);
89+
temp4_vec = svmla_x(SV_TRUE(), temp4_vec, x4_vec, y4_vec);
90+
temp5_vec = svmla_x(SV_TRUE(), temp5_vec, x5_vec, y5_vec);
91+
temp6_vec = svmla_x(SV_TRUE(), temp6_vec, x6_vec, y6_vec);
92+
temp7_vec = svmla_x(SV_TRUE(), temp7_vec, x7_vec, y7_vec);
93+
94+
i += sve_size * 8;
95+
}
96+
97+
if (i < n) {
98+
svbool_t pg0 = SV_WHILE(i + sve_size * 0, n);
99+
svbool_t pg1 = SV_WHILE(i + sve_size * 1, n);
100+
svbool_t pg2 = SV_WHILE(i + sve_size * 2, n);
101+
svbool_t pg3 = SV_WHILE(i + sve_size * 3, n);
102+
svbool_t pg4 = SV_WHILE(i + sve_size * 4, n);
103+
svbool_t pg5 = SV_WHILE(i + sve_size * 5, n);
104+
svbool_t pg6 = SV_WHILE(i + sve_size * 6, n);
105+
svbool_t pg7 = SV_WHILE(i + sve_size * 7, n);
106+
107+
FLOAT *x0_ptr = x + i;
108+
SV_TYPE x0_vec = svld1_vnum(pg0, x0_ptr, 0);
109+
SV_TYPE x1_vec = svld1_vnum(pg1, x0_ptr, 1);
110+
SV_TYPE x2_vec = svld1_vnum(pg2, x0_ptr, 2);
111+
SV_TYPE x3_vec = svld1_vnum(pg3, x0_ptr, 3);
112+
SV_TYPE x4_vec = svld1_vnum(pg4, x0_ptr, 4);
113+
SV_TYPE x5_vec = svld1_vnum(pg5, x0_ptr, 5);
114+
SV_TYPE x6_vec = svld1_vnum(pg6, x0_ptr, 6);
115+
SV_TYPE x7_vec = svld1_vnum(pg7, x0_ptr, 7);
116+
117+
FLOAT *y0_ptr = y + i;
118+
SV_TYPE y0_vec = svld1_vnum(pg0, y0_ptr, 0);
119+
SV_TYPE y1_vec = svld1_vnum(pg1, y0_ptr, 1);
120+
SV_TYPE y2_vec = svld1_vnum(pg2, y0_ptr, 2);
121+
SV_TYPE y3_vec = svld1_vnum(pg3, y0_ptr, 3);
122+
SV_TYPE y4_vec = svld1_vnum(pg4, y0_ptr, 4);
123+
SV_TYPE y5_vec = svld1_vnum(pg5, y0_ptr, 5);
124+
SV_TYPE y6_vec = svld1_vnum(pg6, y0_ptr, 6);
125+
SV_TYPE y7_vec = svld1_vnum(pg7, y0_ptr, 7);
126+
127+
temp0_vec = svmla_m(pg0, temp0_vec, x0_vec, y0_vec);
128+
temp1_vec = svmla_m(pg1, temp1_vec, x1_vec, y1_vec);
129+
temp2_vec = svmla_m(pg2, temp2_vec, x2_vec, y2_vec);
130+
temp3_vec = svmla_m(pg3, temp3_vec, x3_vec, y3_vec);
131+
temp4_vec = svmla_m(pg4, temp4_vec, x4_vec, y4_vec);
132+
temp5_vec = svmla_m(pg5, temp5_vec, x5_vec, y5_vec);
133+
temp6_vec = svmla_m(pg6, temp6_vec, x6_vec, y6_vec);
134+
temp7_vec = svmla_m(pg7, temp7_vec, x7_vec, y7_vec);
135+
}
136+
137+
temp0_vec = svadd_x(SV_TRUE(), temp0_vec, temp1_vec);
138+
temp2_vec = svadd_x(SV_TRUE(), temp2_vec, temp3_vec);
139+
temp4_vec = svadd_x(SV_TRUE(), temp4_vec, temp5_vec);
140+
temp6_vec = svadd_x(SV_TRUE(), temp6_vec, temp7_vec);
141+
temp0_vec = svadd_x(SV_TRUE(), temp0_vec, temp2_vec);
142+
temp4_vec = svadd_x(SV_TRUE(), temp4_vec, temp6_vec);
143+
temp0_vec = svadd_x(SV_TRUE(), temp0_vec, temp4_vec);
144+
145+
return svaddv(SV_TRUE(), temp0_vec);
146+
}

‎kernel/arm64/dot_sve_v8.c

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
/***************************************************************************
2+
Copyright (c) 2025, The OpenBLAS Project
3+
All rights reserved.
4+
5+
Redistribution and use in source and binary forms, with or without
6+
modification, are permitted provided that the following conditions are
7+
met:
8+
9+
1. Redistributions of source code must retain the above copyright
10+
notice, this list of conditions and the following disclaimer.
11+
12+
2. Redistributions in binary form must reproduce the above copyright
13+
notice, this list of conditions and the following disclaimer in
14+
the documentation and/or other materials provided with the
15+
distribution.
16+
3. Neither the name of the OpenBLAS project nor the names of
17+
its contributors may be used to endorse or promote products
18+
derived from this software without specific prior written
19+
permission.
20+
21+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
22+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24+
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
25+
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
27+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
28+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
29+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
30+
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31+
*****************************************************************************/
32+
33+
#define DOT_KERNEL_SVE "dot_kernel_sve_v8.c"
34+
#include "dot.c"

0 commit comments

Comments
 (0)