Skip to content

Commit bb31bbe

Browse files
XiWeiGuXiWeiGu
committed
LoongArch64: Opt somatcopy_ct with LASX
1 parent b371293 commit bb31bbe

File tree

2 files changed

+201
-0
lines changed

2 files changed

+201
-0
lines changed

‎kernel/loongarch64/KERNEL.LA464

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,5 +175,6 @@ DGEMM_SMALL_K_B0_TT = dgemm_small_kernel_tt_lasx.S
175175

176176
SOMATCOPY_RT = somatcopy_rt_lasx.c
177177
SOMATCOPY_RN = somatcopy_rn_lasx.c
178+
SOMATCOPY_CT = somatcopy_ct_lasx.c
178179
SOMATCOPY_CN = somatcopy_cn_lasx.c
179180
endif

‎kernel/loongarch64/somatcopy_ct_lasx.c

Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
/***************************************************************************
2+
Copyright (c) 2024, The OpenBLAS Project
3+
All rights reserved.
4+
Redistribution and use in source and binary forms, with or without
5+
modification, are permitted provided that the following conditions are
6+
met:
7+
1. Redistributions of source code must retain the above copyright
8+
notice, this list of conditions and the following disclaimer.
9+
2. Redistributions in binary form must reproduce the above copyright
10+
notice, this list of conditions and the following disclaimer in
11+
the documentation and/or other materials provided with the
12+
distribution.
13+
3. Neither the name of the OpenBLAS project nor the names of
14+
its contributors may be used to endorse or promote products
15+
derived from this software without specific prior written permission.
16+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19+
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
20+
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
23+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
25+
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26+
*****************************************************************************/
27+
28+
#include "common.h"
29+
30+
#define SAVE1x4(c1) \
31+
"vfmul.s $vr"#c1",$vr"#c1",$vr15;vstelm.w $vr"#c1",%4,0,0;add.d %4,%4,%3;\n\t" \
32+
"vstelm.w $vr"#c1",%4,0,1;add.d %4,%4,%3;vstelm.w $vr"#c1",%4,0,2;add.d %4,%4,%3; \n\t" \
33+
"vstelm.w $vr"#c1",%4,0,3;add.d %4,%4,%3;\n\t"
34+
#define SAVE_2x4(c1, c2, t1, t2) \
35+
"vilvl.w $vr"#t1",$vr"#c2",$vr"#c1";vfmul.s $vr"#t1",$vr"#t1",$vr15; \n\t" \
36+
"vstelm.d $vr"#t1",%4,0,0;add.d %4,%4,%3;vstelm.d $vr"#t1",%4,0,1;add.d %4,%4,%3 \n\t" \
37+
"vilvh.w $vr"#t2",$vr"#c2",$vr"#c1";vfmul.s $vr"#t2",$vr"#t2",$vr15; \n\t" \
38+
"vstelm.d $vr"#t2",%4,0,0;add.d %4,%4,%3;vstelm.d $vr"#t2",%4,0,1;add.d %4,%4,%3 \n\t"
39+
#define SAVE_4x1(b1) \
40+
"vst $vr"#b1",%4,0; add.d %4,%4,%3 \n\t"
41+
#define SAVE_4x2(b1, b2) \
42+
"vst $vr"#b1",%4,0; add.d %4,%4,%3; vst $vr"#b2",%4,0; add.d %4,%4,%3 \n\t"
43+
#define SAVE_4x4(b1, b2, b3, b4) \
44+
"vst $vr"#b1",%4,0; add.d %4,%4,%3; vst $vr"#b2",%4,0; add.d %4,%4,%3 \n\t" \
45+
"vst $vr"#b3",%4,0; add.d %4,%4,%3; vst $vr"#b4",%4,0; add.d %4,%4,%3 \n\t"
46+
#define SAVE_4x8(b1, b2, b3, b4) \
47+
SAVE_4x4(b1, b2, b3, b4) \
48+
"xvpermi.q $xr"#b1",$xr"#b1",1; xvpermi.q $xr"#b2",$xr"#b2",1; xvpermi.q $xr"#b3",$xr"#b3",1; xvpermi.q $xr"#b4",$xr"#b4",1; \n\t" \
49+
SAVE_4x4(b1, b2, b3, b4)
50+
51+
#define TRANS_4x4(a1, a2, a3, a4, t1, t2, t3, t4) \
52+
"vilvl.w $vr"#t1",$vr"#a2",$vr"#a1";vilvh.w $vr"#t2",$vr"#a2",$vr"#a1"; \n\t" \
53+
"vilvl.w $vr"#t3",$vr"#a4",$vr"#a3";vilvh.w $vr"#t4",$vr"#a4",$vr"#a3"; \n\t" \
54+
"vilvl.d $vr"#a1",$vr"#t3",$vr"#t1";vilvh.d $vr"#a2",$vr"#t3",$vr"#t1"; \n\t" \
55+
"vilvl.d $vr"#a3",$vr"#t4",$vr"#t2";vilvh.d $vr"#a4",$vr"#t4",$vr"#t2"; \n\t"
56+
#define TRANS_4x8(a1, a2, a3, a4, t1, t2, t3, t4) \
57+
"xvilvl.w $xr"#t1",$xr"#a2",$xr"#a1"; xvilvh.w $xr"#t2",$xr"#a2",$xr"#a1"; \n\t" \
58+
"xvilvl.w $xr"#t3",$xr"#a4",$xr"#a3"; xvilvh.w $xr"#t4",$xr"#a4",$xr"#a3"; \n\t" \
59+
"xvilvl.d $xr"#a1",$xr"#t3",$xr"#t1"; xvilvh.d $xr"#a2",$xr"#t3",$xr"#t1"; \n\t" \
60+
"xvilvl.d $xr"#a3",$xr"#t4",$xr"#t2"; xvilvh.d $xr"#a4",$xr"#t4",$xr"#t2"; \n\t"
61+
#define COPY_4x16 \
62+
"move %4,%1; addi.d %1,%1,16 \n\t" \
63+
"xvld $xr0,%0,0; xvld $xr4,%0,32; add.d %0,%0,%2; \n\t" \
64+
"xvld $xr1,%0,0; xvld $xr5,%0,32; add.d %0,%0,%2; \n\t" \
65+
"xvld $xr2,%0,0; xvld $xr6,%0,32; add.d %0,%0,%2; \n\t" \
66+
"xvld $xr3,%0,0; xvld $xr7,%0,32; add.d %0,%0,%2; \n\t" \
67+
"xvfmul.s $xr0,$xr0,$xr15;xvfmul.s $xr1,$xr1,$xr15;xvfmul.s $xr2,$xr2,$xr15;xvfmul.s $xr3,$xr3,$xr15 \n\t" \
68+
"xvfmul.s $xr4,$xr4,$xr15;xvfmul.s $xr5,$xr5,$xr15;xvfmul.s $xr6,$xr6,$xr15;xvfmul.s $xr7,$xr7,$xr15 \n\t" \
69+
TRANS_4x8(0,1,2,3,8,9,10,11) SAVE_4x8(0,1,2,3) \
70+
TRANS_4x8(4,5,6,7,8,9,10,11) SAVE_4x8(4,5,6,7)
71+
#define COPY_4x8 \
72+
"move %4,%1; addi.d %1,%1,16 \n\t" \
73+
"xvld $xr0,%0,0; add.d %0,%0,%2; \n\t" \
74+
"xvld $xr1,%0,0; add.d %0,%0,%2; \n\t" \
75+
"xvld $xr2,%0,0; add.d %0,%0,%2; \n\t" \
76+
"xvld $xr3,%0,0; add.d %0,%0,%2; \n\t" \
77+
"xvfmul.s $xr0,$xr0,$xr15;xvfmul.s $xr1,$xr1,$xr15;xvfmul.s $xr2,$xr2,$xr15;xvfmul.s $xr3,$xr3,$xr15 \n\t" \
78+
TRANS_4x8(0,1,2,3,8,9,10,11) SAVE_4x8(0,1,2,3)
79+
#define COPY_4x4 \
80+
"move %4,%1; addi.d %1,%1,16 \n\t" \
81+
"vld $vr0,%0,0; add.d %0,%0,%2; \n\t" \
82+
"vld $vr1,%0,0; add.d %0,%0,%2; \n\t" \
83+
"vld $vr2,%0,0; add.d %0,%0,%2; \n\t" \
84+
"vld $vr3,%0,0; add.d %0,%0,%2; \n\t" \
85+
"vfmul.s $vr0,$vr0,$vr15;vfmul.s $vr1,$vr1,$vr15;vfmul.s $vr2,$vr2,$vr15;vfmul.s $vr3,$vr3,$vr15 \n\t" \
86+
TRANS_4x4(0,1,2,3,8,9,10,11) SAVE_4x4(0,1,2,3)
87+
#define COPY_4x2 \
88+
"move %4,%1; addi.d %1,%1,16 \n\t" \
89+
"vld $vr0,%0,0; add.d %0,%0,%2; \n\t" \
90+
"vld $vr1,%0,0; add.d %0,%0,%2; \n\t" \
91+
"vld $vr2,%0,0; add.d %0,%0,%2; \n\t" \
92+
"vld $vr3,%0,0; add.d %0,%0,%2; \n\t" \
93+
"vfmul.s $vr0,$vr0,$vr15;vfmul.s $vr1,$vr1,$vr15;vfmul.s $vr2,$vr2,$vr15;vfmul.s $vr3,$vr3,$vr15 \n\t" \
94+
TRANS_4x4(0,1,2,3,8,9,10,11) SAVE_4x2(0,1)
95+
#define COPY_4x1 \
96+
"move %4,%1; addi.d %1,%1,16 \n\t" \
97+
"fld.s $f0,%0,0; add.d %0,%0,%2; \n\t" \
98+
"fld.s $f1,%0,0; add.d %0,%0,%2; \n\t" \
99+
"fld.s $f2,%0,0; add.d %0,%0,%2; \n\t" \
100+
"fld.s $f3,%0,0; add.d %0,%0,%2; \n\t" \
101+
"xvinsve0.w $xr0,$xr1,1;xvinsve0.w $xr0,$xr2,2;xvinsve0.w $xr0,$xr3,3; \n\t" \
102+
"vfmul.s $vr0,$vr0,$vr15; \n\t" \
103+
SAVE_4x1(0)
104+
105+
#define COPY_2x16 \
106+
"move %4,%1; addi.d %1,%1,8 \n\t" \
107+
"xvld $xr0,%0,0; xvld $xr2,%0,32; add.d %0,%0,%2; \n\t" \
108+
"xvld $xr1,%0,0; xvld $xr3,%0,32; add.d %0,%0,%2; \n\t" \
109+
"xvpermi.q $xr4,$xr0,1;xvpermi.q $xr6,$xr2,1;xvpermi.q $xr5,$xr1,1;xvpermi.q $xr7,$xr3,1; \n\t" \
110+
SAVE_2x4(0,1,8,9) SAVE_2x4(4,5,8,9) SAVE_2x4(2,3,8,9) SAVE_2x4(6,7,8,9)
111+
#define COPY_2x8 \
112+
"move %4,%1; addi.d %1,%1,8 \n\t" \
113+
"xvld $xr0,%0,0; add.d %0,%0,%2; \n\t" \
114+
"xvld $xr1,%0,0; add.d %0,%0,%2; \n\t" \
115+
"xvpermi.q $xr2,$xr0,1;xvpermi.q $xr3,$xr1,1; \n\t" \
116+
SAVE_2x4(0,1,4,5) SAVE_2x4(2,3,4,5)
117+
#define COPY_2x4 \
118+
"move %4,%1; addi.d %1,%1,8 \n\t" \
119+
"vld $vr0,%0,0; add.d %0,%0,%2; \n\t" \
120+
"vld $vr1,%0,0; add.d %0,%0,%2; \n\t" \
121+
SAVE_2x4(0,1,4,5)
122+
#define COPY_2x2 \
123+
"move %4,%1; addi.d %1,%1,8 \n\t" \
124+
"fld.d $f0,%0,0;add.d %0,%0,%2; \n\t" \
125+
"fld.d $f1,%0,0;add.d %0,%0,%2; \n\t" \
126+
"xvinsve0.d $xr0,$xr1,1;vfmul.s $vr0,$vr0,$vr15;vshuf4i.w $vr0,$vr0,0xd8 \n\t" \
127+
"vstelm.d $vr0,%4,0,0;add.d %4,%4,%3;vstelm.d $vr0,%4,0,1 \n\t"
128+
#define COPY_2x1 \
129+
"move %4,%1; addi.d %1,%1,8 \n\t" \
130+
"fld.s $f0,%0,0;add.d %0,%0,%2; \n\t" \
131+
"fld.s $f1,%0,0;add.d %0,%0,%2; \n\t" \
132+
"xvinsve0.w $xr0,$xr1,1;vfmul.s $vr0,$vr0,$vr15; \n\t" \
133+
"vstelm.d $vr0,%4,0,0; \n\t"
134+
135+
#define COPY_1x16 \
136+
"move %4,%1; addi.d %1,%1,4 \n\t" \
137+
"vld $vr1,%0,0;" SAVE1x4(1) "vld $vr2,%0,16;" SAVE1x4(2) \
138+
"vld $vr1,%0,32;" SAVE1x4(1) "vld $vr2,%0,48;" SAVE1x4(2) \
139+
"add.d %0,%0,%2 \n\t"
140+
#define COPY_1x8 \
141+
"move %4,%1; addi.d %1,%1,4 \n\t" \
142+
"vld $vr1,%0,0;" SAVE1x4(1) "vld $vr2,%0,16;" SAVE1x4(2) \
143+
"add.d %0,%0,%2 \n\t"
144+
#define COPY_1x4 \
145+
"move %4,%1; addi.d %1,%1,4 \n\t" \
146+
"vld $vr1,%0,0;" SAVE1x4(1) \
147+
"add.d %0,%0,%2 \n\t"
148+
#define COPY_1x2 \
149+
"move %4,%1;fld.d $f1,%0,0;add.d %0,%0,%2;vfmul.s $vr1,$vr1,$vr15;vstelm.w $vr1,%4,0,0;add.d %4,%4,%3;vstelm.w $vr1,%4,0,1;\n\t" \
150+
"addi.d %1,%1,4;\n\t"
151+
#define COPY_1x1 \
152+
"fld.s $f1,%0,0;fmul.s $f1,$f1,$f15;fst.s $f1,%1,0;add.d %0,%0,%2;addi.d %1,%1,4;\n\t"
153+
154+
#define ROWS_OF_BLOCK 128
155+
156+
#define COMPUTE(ndim) \
157+
src = src_base; dst = dst_base; \
158+
__asm__ __volatile__( \
159+
"xvldrepl.w $xr15, %6, 0 \n\t" \
160+
"srli.d $r6, %5, 2 \n\t" \
161+
"beqz $r6, "#ndim"3f \n\t" \
162+
#ndim"4: \n\t" \
163+
COPY_4x##ndim \
164+
"addi.d $r6, $r6, -1 \n\t" \
165+
"bnez $r6, "#ndim"4b \n\t" \
166+
#ndim"3: \n\t" \
167+
"andi $r6, %5, 2 \n\t" \
168+
"beqz $r6, "#ndim"1f \n\t" \
169+
#ndim"2: \n\t" \
170+
COPY_2x##ndim \
171+
#ndim"1: \n\t" \
172+
"andi $r6, %5, 1 \n\t" \
173+
"beqz $r6, "#ndim"0f \n\t" \
174+
COPY_1x##ndim \
175+
#ndim"0: \n\t" \
176+
:"+r"(src),"+r"(dst),"+r"(src_ld_bytes),"+r"(dst_ld_bytes),"+r"(dst_tmp) \
177+
:"r"(num_cols),"r"(&ALPHA) \
178+
:"memory", "$r6", "$f0", "$f1", "$f2", "$f3", "$f4", "$f5", "$f6", "$f7", "$f8", "$f9", "$f10", "$f11", "$f15" \
179+
);
180+
181+
int CNAME(BLASLONG rows, BLASLONG cols, FLOAT alpha, FLOAT *a, BLASLONG lda, FLOAT *b, BLASLONG ldb){
182+
float *src, *dst, *dst_tmp=0, *src_base, *dst_base;
183+
uint64_t src_ld_bytes = (uint64_t)lda * sizeof(float), dst_ld_bytes = (uint64_t)ldb * sizeof(float), num_cols = 0;
184+
BLASLONG rows_left, cols_done; float ALPHA = alpha;
185+
if (ALPHA == 0.0) {
186+
dst_base = b;
187+
for (rows_left = rows; rows_left > 0; rows_left--) {memset(dst_base, 0, cols * sizeof(float)); dst_base += ldb;}
188+
return 0;
189+
}
190+
for (cols_done = 0; cols_done < cols; cols_done += num_cols) {
191+
num_cols = cols - cols_done;
192+
if (num_cols > ROWS_OF_BLOCK) num_cols = ROWS_OF_BLOCK;
193+
rows_left = rows; src_base = a + (int64_t)lda * (int64_t)cols_done; dst_base = b + cols_done;
194+
for (;rows_left > 15; rows_left -= 16) {COMPUTE(16) src_base += 16; dst_base += 16 * ldb;}
195+
for (;rows_left > 7; rows_left -= 8) {COMPUTE(8) src_base += 8; dst_base += 8 * ldb;}
196+
for (;rows_left > 3; rows_left -= 4) {COMPUTE(4) src_base += 4; dst_base += 4 * ldb;}
197+
for (;rows_left > 1; rows_left -= 2) {COMPUTE(2) src_base += 2; dst_base += 2 * ldb;}
198+
if (rows_left > 0) {COMPUTE(1) src_base ++; dst_base += ldb;}
199+
}
200+
}

0 commit comments

Comments
 (0)