Skip to content

Commit 016bdb9

Browse files
martin-frbgmartin-frbg
authored
Merge pull request #4946 from XiWeiGu/la64_omatcopy_lasx
LoongArch64: Opt somatcopy with LASX
2 parents a938972 + ffaa576 commit 016bdb9

File tree

7 files changed

+951
-1
lines changed

7 files changed

+951
-1
lines changed

‎benchmark/Makefile

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,7 @@ goto :: slinpack.goto dlinpack.goto clinpack.goto zlinpack.goto \
103103
sgetri.goto dgetri.goto cgetri.goto zgetri.goto \
104104
spotrf.goto dpotrf.goto cpotrf.goto zpotrf.goto \
105105
ssymm.goto dsymm.goto csymm.goto zsymm.goto \
106+
somatcopy.goto domatcopy.goto comatcopy.goto zomatcopy.goto \
106107
saxpby.goto daxpby.goto caxpby.goto zaxpby.goto $(GOTO_HALF_TARGETS)
107108

108109
acml :: slinpack.acml dlinpack.acml clinpack.acml zlinpack.acml \
@@ -276,6 +277,7 @@ goto :: sgemm.goto dgemm.goto cgemm.goto zgemm.goto \
276277
samin.goto damin.goto camin.goto zamin.goto \
277278
smin.goto dmin.goto \
278279
saxpby.goto daxpby.goto caxpby.goto zaxpby.goto \
280+
somatcopy.goto domatcopy.goto comatcopy.goto zomatcopy.goto \
279281
snrm2.goto dnrm2.goto scnrm2.goto dznrm2.goto $(GOTO_LAPACK_TARGETS) $(GOTO_HALF_TARGETS)
280282

281283
acml :: slinpack.acml dlinpack.acml clinpack.acml zlinpack.acml \
@@ -2906,6 +2908,29 @@ dznrm2.goto : dznrm2.$(SUFFIX) ../$(LIBNAME)
29062908
dznrm2.atlas : dznrm2.$(SUFFIX)
29072909
-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBATLAS) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
29082910

2911+
###################################################################################################
2912+
2913+
############################################ SOMATCOPY ############################################
2914+
somatcopy.goto : somatcopy.$(SUFFIX) ../$(LIBNAME)
2915+
$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB) -lm
2916+
2917+
###################################################################################################
2918+
2919+
############################################ DOMATCOPY ############################################
2920+
domatcopy.goto : domatcopy.$(SUFFIX) ../$(LIBNAME)
2921+
$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB) -lm
2922+
2923+
###################################################################################################
2924+
2925+
############################################ COMATCOPY ############################################
2926+
comatcopy.goto : comatcopy.$(SUFFIX) ../$(LIBNAME)
2927+
$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB) -lm
2928+
2929+
###################################################################################################
2930+
2931+
############################################ ZOMATCOPY ############################################
2932+
zomatcopy.goto : zomatcopy.$(SUFFIX) ../$(LIBNAME)
2933+
$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB) -lm
29092934

29102935
###################################################################################################
29112936

@@ -3435,11 +3460,23 @@ scnrm2.$(SUFFIX) : nrm2.c
34353460
dznrm2.$(SUFFIX) : nrm2.c
34363461
$(CC) $(CFLAGS) -c -DCOMPLEX -DDOUBLE -o $(@F) $^
34373462

3463+
somatcopy.$(SUFFIX) : omatcopy.c
3464+
$(CC) $(CFLAGS) -c -UCOMPLEX -UDOUBLE -o $(@F) $^
3465+
3466+
domatcopy.$(SUFFIX) : omatcopy.c
3467+
$(CC) $(CFLAGS) -c -UCOMPLEX -DDOUBLE -o $(@F) $^
3468+
3469+
comatcopy.$(SUFFIX) : omatcopy.c
3470+
$(CC) $(CFLAGS) -c -DCOMPLEX -UDOUBLE -o $(@F) $^
3471+
3472+
zomatcopy.$(SUFFIX) : omatcopy.c
3473+
$(CC) $(CFLAGS) -c -DCOMPLEX -DDOUBLE -o $(@F) $^
3474+
34383475

34393476
smallscaling: smallscaling.c ../$(LIBNAME)
34403477
$(CC) $(CFLAGS) -o $(@F) $^ $(EXTRALIB) -fopenmp -lm -lpthread
34413478

34423479
clean ::
34433480
@rm -f *.goto *.mkl *.acml *.atlas *.veclib *.essl smallscaling
34443481

3445-
include $(TOPDIR)/Makefile.tail
3482+
include $(TOPDIR)/Makefile.tail

‎benchmark/omatcopy.c

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
/***************************************************************************
2+
Copyright (c) 2024, The OpenBLAS Project
3+
All rights reserved.
4+
Redistribution and use in source and binary forms, with or without
5+
modification, are permitted provided that the following conditions are
6+
met:
7+
1. Redistributions of source code must retain the above copyright
8+
notice, this list of conditions and the following disclaimer.
9+
2. Redistributions in binary form must reproduce the above copyright
10+
notice, this list of conditions and the following disclaimer in
11+
the documentation and/or other materials provided with the
12+
distribution.
13+
3. Neither the name of the OpenBLAS project nor the names of
14+
its contributors may be used to endorse or promote products
15+
derived from this software without specific prior written permission.
16+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19+
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
20+
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
23+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
25+
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26+
*****************************************************************************/
27+
28+
#include "bench.h"
29+
30+
#undef OMATCOPY
31+
32+
#ifndef COMPLEX
33+
#ifdef DOUBLE
34+
#define OMATCOPY BLASFUNC(domatcopy)
35+
#else
36+
#define OMATCOPY BLASFUNC(somatcopy)
37+
#endif
38+
#else
39+
#ifdef DOUBLE
40+
#define OMATCOPY BLASFUNC(zomatcopy)
41+
#else
42+
#define OMATCOPY BLASFUNC(comatcopy)
43+
#endif
44+
#endif
45+
int main(int argc, char *argv[]){
46+
FLOAT *a, *b;
47+
FLOAT alpha[] = {1.0, 0.0};
48+
char trans = 'N';
49+
char order = 'C';
50+
blasint crows, ccols, clda, cldb;
51+
int loops = 1;
52+
char *p;
53+
54+
int from = 1;
55+
int to = 200;
56+
int step = 1;
57+
int i, j;
58+
59+
double time1, timeg;
60+
61+
argc--;argv++;
62+
63+
if (argc > 0) { from = atol(*argv); argc--; argv++; }
64+
if (argc > 0) { to = MAX(atol(*argv), from); argc--; argv++; }
65+
if (argc > 0) { step = atol(*argv); argc--; argv++; }
66+
67+
if ((p = getenv("OPENBLAS_TRANS"))) {
68+
trans=*p;
69+
}
70+
if ((p = getenv("OPENBLAS_ORDER"))) {
71+
order=*p;
72+
}
73+
TOUPPER(trans);
74+
TOUPPER(order);
75+
fprintf(stderr, "From : %3d To : %3d Step=%d : Trans=%c : Order=%c\n", from, to, step, trans, order);
76+
p = getenv("OPENBLAS_LOOPS");
77+
if ( p != NULL ) {
78+
loops = atoi(p);
79+
}
80+
81+
if (( a = (FLOAT *)malloc(sizeof(FLOAT) * to * to * COMPSIZE)) == NULL) {
82+
fprintf(stderr,"Out of Memory!!\n");exit(1);
83+
}
84+
if (( b = (FLOAT *)malloc(sizeof(FLOAT) * to * to * COMPSIZE)) == NULL) {
85+
fprintf(stderr,"Out of Memory!!\n");exit(1);
86+
}
87+
88+
#ifdef __linux
89+
srandom(getpid());
90+
#endif
91+
92+
for (i = 0; i < to * to * COMPSIZE; i++) {
93+
a[i] = ((FLOAT) rand() / (FLOAT) RAND_MAX) - 0.5;
94+
}
95+
for (i = 0; i < to * to * COMPSIZE; i++) {
96+
b[i] = ((FLOAT) rand() / (FLOAT) RAND_MAX) - 0.5;
97+
}
98+
99+
fprintf(stderr, " SIZE Flops Time\n");
100+
for (i = from; i <= to; i += step) {
101+
cldb = clda = crows = ccols = i;
102+
fprintf(stderr, " ROWS=%4d, COLS=%4d : ", (int)crows, (int)ccols);
103+
begin();
104+
105+
for (j=0; j<loops; j++) {
106+
OMATCOPY (&order, &trans, &crows, &ccols, alpha, a, &clda, b, &cldb);
107+
}
108+
109+
end();
110+
time1 = getsec();
111+
112+
timeg = time1/loops;
113+
fprintf(stderr,
114+
" %10.2f MFlops %10.6f sec\n",
115+
COMPSIZE * COMPSIZE * (double)ccols * (double)crows / timeg * 1.e-6, time1);
116+
}
117+
118+
free(a);
119+
free(b);
120+
121+
return 0;
122+
}

‎kernel/loongarch64/KERNEL.LA464

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,4 +172,9 @@ DGEMM_SMALL_K_TN = dgemm_small_kernel_tn_lasx.S
172172
DGEMM_SMALL_K_B0_TN = dgemm_small_kernel_tn_lasx.S
173173
DGEMM_SMALL_K_TT = dgemm_small_kernel_tt_lasx.S
174174
DGEMM_SMALL_K_B0_TT = dgemm_small_kernel_tt_lasx.S
175+
176+
SOMATCOPY_RT = somatcopy_rt_lasx.c
177+
SOMATCOPY_RN = somatcopy_rn_lasx.c
178+
SOMATCOPY_CT = somatcopy_ct_lasx.c
179+
SOMATCOPY_CN = somatcopy_cn_lasx.c
175180
endif

‎kernel/loongarch64/somatcopy_cn_lasx.c

Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
/***************************************************************************
2+
Copyright (c) 2024, The OpenBLAS Project
3+
All rights reserved.
4+
Redistribution and use in source and binary forms, with or without
5+
modification, are permitted provided that the following conditions are
6+
met:
7+
1. Redistributions of source code must retain the above copyright
8+
notice, this list of conditions and the following disclaimer.
9+
2. Redistributions in binary form must reproduce the above copyright
10+
notice, this list of conditions and the following disclaimer in
11+
the documentation and/or other materials provided with the
12+
distribution.
13+
3. Neither the name of the OpenBLAS project nor the names of
14+
its contributors may be used to endorse or promote products
15+
derived from this software without specific prior written permission.
16+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19+
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
20+
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
23+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
25+
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26+
*****************************************************************************/
27+
28+
#include "common.h"
29+
#define SAVE_4x1(b1, b2, b3, b4) \
30+
"vstelm.w $vr"#b1",%1,0,0;add.d %1,%1,%3;vstelm.w $vr"#b2",%1,0,0;add.d %1,%1,%3;\n\t" \
31+
"vstelm.w $vr"#b3",%1,0,0;add.d %1,%1,%3;vstelm.w $vr"#b4",%1,0,0;add.d %1,%1,%3;\n\t"
32+
#define SAVE_4x2(b1, b2, b3, b4) \
33+
"vstelm.d $vr"#b1",%1,0,0;add.d %1,%1,%3;vstelm.d $vr"#b2",%1,0,0;add.d %1,%1,%3;\n\t" \
34+
"vstelm.d $vr"#b3",%1,0,0;add.d %1,%1,%3;vstelm.d $vr"#b4",%1,0,0;add.d %1,%1,%3;\n\t"
35+
#define SAVE_4x4(b1, b2, b3, b4) \
36+
"vst $vr"#b1",%1,0;add.d %1,%1,%3;vst $vr"#b2",%1,0;add.d %1,%1,%3;\n\t" \
37+
"vst $vr"#b3",%1,0;add.d %1,%1,%3;vst $vr"#b4",%1,0;add.d %1,%1,%3;\n\t"
38+
#define SAVE_4x8(b1, b2, b3, b4) \
39+
"xvst $xr"#b1",%1,0;add.d %1,%1,%3;xvst $xr"#b2",%1,0;add.d %1,%1,%3;\n\t" \
40+
"xvst $xr"#b3",%1,0;add.d %1,%1,%3;xvst $xr"#b4",%1,0;add.d %1,%1,%3;\n\t"
41+
#define SAVE_4x16(b1, b2, b3, b4, b5, b6, b7, b8) \
42+
"xvst $xr"#b1",%1,0;xvst $xr"#b2",%1,32;add.d %1,%1,%3;\n\t" \
43+
"xvst $xr"#b3",%1,0;xvst $xr"#b4",%1,32;add.d %1,%1,%3;\n\t" \
44+
"xvst $xr"#b5",%1,0;xvst $xr"#b6",%1,32;add.d %1,%1,%3;\n\t" \
45+
"xvst $xr"#b7",%1,0;xvst $xr"#b8",%1,32;add.d %1,%1,%3;\n\t"
46+
#define SAVE_2x16(b1, b2, b3, b4) \
47+
"xvst $xr"#b1",%1,0;xvst $xr"#b2",%1,32;add.d %1,%1,%3;\n\t" \
48+
"xvst $xr"#b3",%1,0;xvst $xr"#b4",%1,32;add.d %1,%1,%3;\n\t"
49+
#define SAVE_2x8(b1, b2) \
50+
"xvst $xr"#b1",%1,0;add.d %1,%1,%3;xvst $xr"#b2",%1,0;add.d %1,%1,%3;\n\t"
51+
#define SAVE_2x4(b1, b2) \
52+
"vst $vr"#b1",%1,0;add.d %1,%1,%3;vst $vr"#b2",%1,0;add.d %1,%1,%3;\n\t"
53+
#define SAVE_2x2(b1, b2) \
54+
"vstelm.d $vr"#b1",%1,0,0;add.d %1,%1,%3;vstelm.d $vr"#b2",%1,0,0;add.d %1,%1,%3;\n\t"
55+
#define SAVE_2x1(b1, b2) \
56+
"vstelm.w $vr"#b1",%1,0,0;add.d %1,%1,%3;vstelm.w $vr"#b2",%1,0,0;add.d %1,%1,%3;\n\t"
57+
#define SAVE_1x16(b1, b2) \
58+
"xvst $xr"#b1",%1,0;xvst $xr"#b2",%1,32;add.d %1,%1,%3;\n\t"
59+
#define SAVE_1x8(b1) \
60+
"xvst $xr"#b1",%1,0;add.d %1,%1,%3;\n\t"
61+
#define SAVE_1x4(b1) \
62+
"vst $vr"#b1",%1,0;add.d %1,%1,%3;\n\t"
63+
#define SAVE_1x2(b1) \
64+
"vstelm.d $vr"#b1",%1,0,0;add.d %1,%1,%3;\n\t"
65+
#define SAVE_1x1(b1) \
66+
"vstelm.w $vr"#b1",%1,0,0;add.d %1,%1,%3;\n\t"
67+
#define COPY_4x16 \
68+
"xvld $xr0,%0,0; xvld $xr4,%0,32; add.d %0,%0,%2; \n\t" \
69+
"xvld $xr1,%0,0; xvld $xr5,%0,32; add.d %0,%0,%2; \n\t" \
70+
"xvld $xr2,%0,0; xvld $xr6,%0,32; add.d %0,%0,%2; \n\t" \
71+
"xvld $xr3,%0,0; xvld $xr7,%0,32; add.d %0,%0,%2; \n\t" \
72+
"xvfmul.s $xr0,$xr0,$xr15;xvfmul.s $xr1,$xr1,$xr15;xvfmul.s $xr2,$xr2,$xr15;xvfmul.s $xr3,$xr3,$xr15 \n\t" \
73+
"xvfmul.s $xr4,$xr4,$xr15;xvfmul.s $xr5,$xr5,$xr15;xvfmul.s $xr6,$xr6,$xr15;xvfmul.s $xr7,$xr7,$xr15 \n\t" \
74+
SAVE_4x16(0,4,1,5,2,6,3,7)
75+
#define COPY_4x8 \
76+
"xvld $xr0,%0,0; add.d %0,%0,%2; \n\t" \
77+
"xvld $xr1,%0,0; add.d %0,%0,%2; \n\t" \
78+
"xvld $xr2,%0,0; add.d %0,%0,%2; \n\t" \
79+
"xvld $xr3,%0,0; add.d %0,%0,%2; \n\t" \
80+
"xvfmul.s $xr0,$xr0,$xr15;xvfmul.s $xr1,$xr1,$xr15;xvfmul.s $xr2,$xr2,$xr15;xvfmul.s $xr3,$xr3,$xr15 \n\t" \
81+
SAVE_4x8(0,1,2,3)
82+
#define COPY_4x4 \
83+
"vld $vr0,%0,0; add.d %0,%0,%2; \n\t" \
84+
"vld $vr1,%0,0; add.d %0,%0,%2; \n\t" \
85+
"vld $vr2,%0,0; add.d %0,%0,%2; \n\t" \
86+
"vld $vr3,%0,0; add.d %0,%0,%2; \n\t" \
87+
"vfmul.s $vr0,$vr0,$vr15;vfmul.s $vr1,$vr1,$vr15;vfmul.s $vr2,$vr2,$vr15;vfmul.s $vr3,$vr3,$vr15 \n\t" \
88+
SAVE_4x4(0,1,2,3)
89+
#define COPY_4x2 \
90+
"vld $vr0,%0,0; add.d %0,%0,%2; \n\t" \
91+
"vld $vr1,%0,0; add.d %0,%0,%2; \n\t" \
92+
"vld $vr2,%0,0; add.d %0,%0,%2; \n\t" \
93+
"vld $vr3,%0,0; add.d %0,%0,%2; \n\t" \
94+
"vfmul.s $vr0,$vr0,$vr15;vfmul.s $vr1,$vr1,$vr15;vfmul.s $vr2,$vr2,$vr15;vfmul.s $vr3,$vr3,$vr15 \n\t" \
95+
SAVE_4x2(0,1,2,3)
96+
#define COPY_4x1 \
97+
"fld.s $f0,%0,0; add.d %0,%0,%2; \n\t" \
98+
"fld.s $f1,%0,0; add.d %0,%0,%2; \n\t" \
99+
"fld.s $f2,%0,0; add.d %0,%0,%2; \n\t" \
100+
"fld.s $f3,%0,0; add.d %0,%0,%2; \n\t" \
101+
"fmul.s $f0,$f0,$f15;fmul.s $f1,$f1,$f15;fmul.s $f2,$f2,$f15;fmul.s $f3,$f3,$f15 \n\t" \
102+
SAVE_4x1(0,1,2,3)
103+
#define COPY_2x16 \
104+
"xvld $xr0,%0,0; xvld $xr2,%0,32; add.d %0,%0,%2; \n\t" \
105+
"xvld $xr1,%0,0; xvld $xr3,%0,32; add.d %0,%0,%2; \n\t" \
106+
"xvfmul.s $xr0,$xr0,$xr15;xvfmul.s $xr1,$xr1,$xr15;xvfmul.s $xr2,$xr2,$xr15;xvfmul.s $xr3,$xr3,$xr15 \n\t" \
107+
SAVE_2x16(0,2,1,3)
108+
#define COPY_2x8 \
109+
"xvld $xr0,%0,0; add.d %0,%0,%2; \n\t" \
110+
"xvld $xr1,%0,0; add.d %0,%0,%2; \n\t" \
111+
"xvfmul.s $xr0,$xr0,$xr15;xvfmul.s $xr1,$xr1,$xr15; \n\t" \
112+
SAVE_2x8(0,1)
113+
#define COPY_2x4 \
114+
"vld $vr0,%0,0; add.d %0,%0,%2; \n\t" \
115+
"vld $vr1,%0,0; add.d %0,%0,%2; \n\t" \
116+
"vfmul.s $vr0,$vr0,$vr15;vfmul.s $vr1,$vr1,$vr15; \n\t" \
117+
SAVE_2x4(0,1)
118+
#define COPY_2x2 \
119+
"fld.d $f0,%0,0;add.d %0,%0,%2; \n\t" \
120+
"fld.d $f1,%0,0;add.d %0,%0,%2; \n\t" \
121+
"vfmul.s $vr0,$vr0,$vr15;vfmul.s $vr1,$vr1,$vr15; \n\t" \
122+
SAVE_2x2(0,1)
123+
#define COPY_2x1 \
124+
"fld.s $f0,%0,0;add.d %0,%0,%2; \n\t" \
125+
"fld.s $f1,%0,0;add.d %0,%0,%2; \n\t" \
126+
"fmul.s $f0,$f0,$f15;fmul.s $f1,$f1,$f15; \n\t" \
127+
SAVE_2x1(0,1)
128+
#define COPY_1x16 \
129+
"xvld $xr0,%0,0; xvld $xr1,%0,32; add.d %0,%0,%2; \n\t" \
130+
"xvfmul.s $xr0,$xr0,$xr15;xvfmul.s $xr1,$xr1,$xr15; \n\t" \
131+
SAVE_1x16(0,1)
132+
#define COPY_1x8 \
133+
"xvld $xr0,%0,0; add.d %0,%0,%2; \n\t" \
134+
"xvfmul.s $xr0,$xr0,$xr15; \n\t" \
135+
SAVE_1x8(0)
136+
#define COPY_1x4 \
137+
"vld $vr0,%0,0; add.d %0,%0,%2; \n\t" \
138+
"vfmul.s $vr0,$vr0,$vr15; \n\t" \
139+
SAVE_1x4(0)
140+
#define COPY_1x2 \
141+
"fld.d $f0,%0,0;add.d %0,%0,%2; \n\t" \
142+
"vfmul.s $vr0,$vr0,$vr15; \n\t" \
143+
SAVE_1x2(0)
144+
#define COPY_1x1 \
145+
"fld.s $f0,%0,0;add.d %0,%0,%2; \n\t" \
146+
"fmul.s $f0,$f0,$f15; \n\t" \
147+
SAVE_1x1(0)
148+
#define ROWS_OF_BLOCK 128
149+
#define COMPUTE(ndim) \
150+
src = src_base; dst = dst_base; \
151+
__asm__ __volatile__( \
152+
"xvldrepl.w $xr15, %6, 0 \n\t" \
153+
"srli.d $r6, %5, 2 \n\t" \
154+
"beqz $r6, "#ndim"3f \n\t" \
155+
#ndim"4: \n\t" \
156+
COPY_4x##ndim \
157+
"addi.d $r6, $r6, -1 \n\t" \
158+
"bnez $r6, "#ndim"4b \n\t" \
159+
#ndim"3: \n\t" \
160+
"andi $r6, %5, 2 \n\t" \
161+
"beqz $r6, "#ndim"1f \n\t" \
162+
#ndim"2: \n\t" \
163+
COPY_2x##ndim \
164+
#ndim"1: \n\t" \
165+
"andi $r6, %5, 1 \n\t" \
166+
"beqz $r6, "#ndim"0f \n\t" \
167+
COPY_1x##ndim \
168+
#ndim"0: \n\t" \
169+
:"+r"(src),"+r"(dst),"+r"(src_ld_bytes),"+r"(dst_ld_bytes),"+r"(dst_tmp) \
170+
:"r"(num_cols),"r"(&ALPHA) \
171+
:"memory", "$r6", "$f0", "$f1", "$f2", "$f3", "$f4", "$f5", "$f6", "$f7", "$f8", "$f9", "$f10", "$f11", "$f15" \
172+
);
173+
174+
int CNAME(BLASLONG rows, BLASLONG cols, FLOAT alpha, FLOAT *a, BLASLONG lda, FLOAT *b, BLASLONG ldb){
175+
float *src, *dst, *dst_tmp=0, *src_base, *dst_base;
176+
uint64_t src_ld_bytes = (uint64_t)lda * sizeof(float), dst_ld_bytes = (uint64_t)ldb * sizeof(float), num_cols = 0;
177+
BLASLONG rows_left, cols_done; float ALPHA = alpha;
178+
if (ALPHA == 0.0) {
179+
dst_base = b;
180+
for (rows_left = rows; rows_left > 0; rows_left--) {memset(dst_base, 0, cols * sizeof(float)); dst_base += ldb;}
181+
return 0;
182+
}
183+
for (cols_done = 0; cols_done < cols; cols_done += num_cols) {
184+
num_cols = cols - cols_done;
185+
if (num_cols > ROWS_OF_BLOCK) num_cols = ROWS_OF_BLOCK;
186+
rows_left = rows; src_base = a + (int64_t)lda * (int64_t)cols_done; dst_base = b + cols_done;
187+
for (;rows_left > 15; rows_left -= 16) {COMPUTE(16) src_base += 16; dst_base += 16;}
188+
for (;rows_left > 7; rows_left -= 8) {COMPUTE(8) src_base += 8; dst_base += 8;}
189+
for (;rows_left > 3; rows_left -= 4) {COMPUTE(4) src_base += 4; dst_base += 4;}
190+
for (;rows_left > 1; rows_left -= 2) {COMPUTE(2) src_base += 2; dst_base += 2;}
191+
if (rows_left > 0) {COMPUTE(1) src_base ++; dst_base ++;}
192+
}
193+
}

0 commit comments

Comments
 (0)