Skip to content

Commit a97b301

Browse files
cgemm/ctrmm power9
1 parent cdbfb89 commit a97b301

File tree

6 files changed

+6134
-6
lines changed

6 files changed

+6134
-6
lines changed

kernel/power/KERNEL.POWER9

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55

66
STRMMKERNEL = sgemm_kernel_power9.S
77
DTRMMKERNEL = dgemm_kernel_power9.S
8-
CTRMMKERNEL = ctrmm_kernel_8x4_power8.S
8+
CTRMMKERNEL = cgemm_kernel_power9.S
99
ZTRMMKERNEL = zgemm_kernel_power9.S
1010

1111
SGEMMKERNEL = sgemm_kernel_power9.S
@@ -28,9 +28,9 @@ DGEMMITCOPYOBJ = dgemm_itcopy.o
2828
DGEMMONCOPYOBJ = dgemm_oncopy.o
2929
DGEMMOTCOPYOBJ = dgemm_otcopy.o
3030

31-
CGEMMKERNEL = cgemm_kernel_8x4_power8.S
31+
CGEMMKERNEL = cgemm_kernel_power9.S
3232
CGEMMINCOPY = ../generic/zgemm_ncopy_8.c
33-
CGEMMITCOPY = cgemm_tcopy_8_power8.S
33+
CGEMMITCOPY = ../generic/zgemm_tcopy_8.c
3434
CGEMMONCOPY = ../generic/zgemm_ncopy_4.c
3535
CGEMMOTCOPY = ../generic/zgemm_tcopy_4.c
3636
CGEMMONCOPYOBJ = cgemm_oncopy.o

kernel/power/cgemm_kernel_power9.S

Lines changed: 293 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,293 @@
1+
/***************************************************************************
2+
Copyright (c) 2013-2019, The OpenBLAS Project
3+
All rights reserved.
4+
Redistribution and use in source and binary forms, with or without
5+
modification, are permitted provided that the following conditions are
6+
met:
7+
1. Redistributions of source code must retain the above copyright
8+
notice, this list of conditions and the following disclaimer.
9+
2. Redistributions in binary form must reproduce the above copyright
10+
notice, this list of conditions and the following disclaimer in
11+
the documentation and/or other materials provided with the
12+
distribution.
13+
3. Neither the name of the OpenBLAS project nor the names of
14+
its contributors may be used to endorse or promote products
15+
derived from this software without specific prior written permission.
16+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19+
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
20+
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
23+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
25+
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26+
*****************************************************************************/
27+
28+
/**************************************************************************************
29+
* Abdelrauf(quickwritereader@gmail.com)
30+
* BLASTEST : OK
31+
* CTEST : OK
32+
* TEST : OK
33+
* LAPACK-TEST : OK
34+
**************************************************************************************/
35+
#define ASSEMBLER
36+
#include "common.h"
37+
#include "def_vsx.h"
38+
39+
40+
#define LOAD ld
41+
#define STACKSIZE (512 )
42+
#define FLINK_SAVE (STACKSIZE+16) /* 16($r12) */
43+
#define M r3
44+
#define N r4
45+
#define K r5
46+
47+
48+
#define A r8
49+
#define B r9
50+
#define C r10
51+
#define LDC r6
52+
#define OFFSET r7
53+
54+
55+
#define alpha_r vs19
56+
#define alpha_i vs20
57+
#define save_permute_1 vs21
58+
#define permute_mask vs22
59+
#define o0 0
60+
61+
62+
#define T1 r11
63+
#define T2 r12
64+
#define T3 r14
65+
#define T4 r15
66+
#define T5 r16
67+
#define T6 r17
68+
#define L r18
69+
#define T7 r19
70+
#define T8 r20
71+
#define TEMP_REG r21
72+
#define I r22
73+
#define J r23
74+
#define AO r24
75+
#define BO r25
76+
#define CO r26
77+
#define T9 r27
78+
#define T10 r28
79+
#define PRE r29
80+
81+
#define T12 r30
82+
#define T13 r31
83+
84+
#include "cgemm_macros_power9.S"
85+
86+
.equ perm_const1, 0x0405060700010203
87+
.equ perm_const2, 0x0c0d0e0f08090a0b
88+
.equ save_permute_12, 0x0c0d0e0f1c1d1e1f
89+
.equ save_permute_11, 0x0405060714151617
90+
91+
92+
93+
#ifndef NEEDPARAM
94+
95+
PROLOGUE
96+
PROFCODE
97+
98+
99+
addi SP, SP, -STACKSIZE
100+
mflr r0
101+
102+
103+
stfd f14, 0(SP)
104+
stfd f15, 8(SP)
105+
stfd f16, 16(SP)
106+
stfd f17, 24(SP)
107+
108+
stfd f18, 32(SP)
109+
stfd f19, 40(SP)
110+
stfd f20, 48(SP)
111+
stfd f21, 56(SP)
112+
113+
stfd f22, 64(SP)
114+
stfd f23, 72(SP)
115+
stfd f24, 80(SP)
116+
stfd f25, 88(SP)
117+
118+
stfd f26, 96(SP)
119+
stfd f27, 104(SP)
120+
stfd f28, 112(SP)
121+
stfd f29, 120(SP)
122+
123+
stfd f30, 128(SP)
124+
stfd f31, 136(SP)
125+
126+
127+
std r31, 144(SP)
128+
std r30, 152(SP)
129+
std r29, 160(SP)
130+
std r28, 168(SP)
131+
std r27, 176(SP)
132+
std r26, 184(SP)
133+
std r25, 192(SP)
134+
std r24, 200(SP)
135+
std r23, 208(SP)
136+
std r22, 216(SP)
137+
std r21, 224(SP)
138+
std r20, 232(SP)
139+
std r19, 240(SP)
140+
std r18, 248(SP)
141+
std r17, 256(SP)
142+
std r16, 264(SP)
143+
std r15, 272(SP)
144+
std r14, 280(SP)
145+
146+
147+
stxv vs52, 288(SP)
148+
stxv vs53, 304(SP)
149+
stxv vs54, 320(SP)
150+
stxv vs55, 336(SP)
151+
stxv vs56, 352(SP)
152+
stxv vs57, 368(SP)
153+
stxv vs58, 384(SP)
154+
stxv vs59, 400(SP)
155+
stxv vs60, 416(SP)
156+
stxv vs61, 432(SP)
157+
stxv vs62, 448(SP)
158+
stxv vs63, 464(SP)
159+
std r0, FLINK_SAVE(SP)
160+
161+
162+
163+
ld LDC, FRAMESLOT(0) + STACKSIZE(SP)
164+
165+
166+
167+
#ifdef TRMMKERNEL
168+
ld OFFSET, FRAMESLOT(1) + STACKSIZE(SP)
169+
#endif
170+
slwi LDC, LDC, ZBASE_SHIFT
171+
172+
173+
174+
/*alpha is stored in f1. convert to single and splat*/
175+
xscvdpspn alpha_r,vs1
176+
xscvdpspn alpha_i,vs2
177+
xxspltw alpha_r,alpha_r,0
178+
xxspltw alpha_i,alpha_i,0
179+
/*load reverse permute mask for big endian
180+
uint128 = 0xc0d0e0f08090a0b0405060700010203
181+
*/
182+
183+
lis T2, perm_const2@highest
184+
lis T1, perm_const1@highest
185+
lis T3, save_permute_12@highest
186+
lis T4, save_permute_11@highest
187+
188+
189+
ori T2, T2, perm_const2@higher
190+
ori T1, T1, perm_const1@higher
191+
ori T3, T3, save_permute_12@higher
192+
ori T4, T4, save_permute_11@higher
193+
194+
195+
rldicr T2, T2, 32, 31
196+
rldicr T1, T1, 32, 31
197+
rldicr T3, T3, 32, 31
198+
rldicr T4, T4, 32, 31
199+
200+
oris T2, T2, perm_const2@h
201+
oris T1, T1, perm_const1@h
202+
oris T3, T3, save_permute_12@h
203+
oris T4, T4, save_permute_11@h
204+
205+
206+
ori T2, T2, perm_const2@l
207+
ori T1, T1, perm_const1@l
208+
ori T3, T3, save_permute_12@l
209+
ori T4, T4, save_permute_11@l
210+
211+
212+
li r0,0
213+
li PRE,512
214+
215+
#if defined(CC) || defined(CR) || defined(RC) || defined(RR)
216+
/*negate for this case as we will use addition -1*(a+b) */
217+
xvnegsp alpha_r,alpha_r
218+
xvnegsp alpha_i,alpha_i
219+
#endif
220+
221+
mtvsrdd permute_mask,T2,T1
222+
mtvsrdd save_permute_1,T3,T4
223+
224+
/*mask is reverse permute so we have to make it inner permute */
225+
xxpermdi permute_mask, permute_mask, permute_mask,2
226+
227+
#include "cgemm_logic_power9.S"
228+
229+
.L999:
230+
lfd f14, 0(SP)
231+
lfd f15, 8(SP)
232+
lfd f16, 16(SP)
233+
lfd f17, 24(SP)
234+
235+
lfd f18, 32(SP)
236+
lfd f19, 40(SP)
237+
lfd f20, 48(SP)
238+
lfd f21, 56(SP)
239+
240+
lfd f22, 64(SP)
241+
lfd f23, 72(SP)
242+
lfd f24, 80(SP)
243+
lfd f25, 88(SP)
244+
245+
lfd f26, 96(SP)
246+
lfd f27, 104(SP)
247+
lfd f28, 112(SP)
248+
lfd f29, 120(SP)
249+
250+
lfd f30, 128(SP)
251+
lfd f31, 136(SP)
252+
253+
ld r31, 144(SP)
254+
ld r30, 152(SP)
255+
ld r29, 160(SP)
256+
ld r28, 168(SP)
257+
ld r27, 176(SP)
258+
ld r26, 184(SP)
259+
ld r25, 192(SP)
260+
ld r24, 200(SP)
261+
ld r23, 208(SP)
262+
ld r22, 216(SP)
263+
ld r21, 224(SP)
264+
ld r20, 232(SP)
265+
ld r19, 240(SP)
266+
ld r18, 248(SP)
267+
ld r17, 256(SP)
268+
ld r16, 264(SP)
269+
ld r15, 272(SP)
270+
ld r14, 280(SP)
271+
272+
ld r0, FLINK_SAVE(SP)
273+
274+
lxv vs52, 288(SP)
275+
lxv vs53, 304(SP)
276+
lxv vs54, 320(SP)
277+
lxv vs55, 336(SP)
278+
lxv vs56, 352(SP)
279+
lxv vs57, 368(SP)
280+
lxv vs58, 384(SP)
281+
lxv vs59, 400(SP)
282+
mtlr r0
283+
lxv vs60, 416(SP)
284+
lxv vs61, 432(SP)
285+
lxv vs62, 448(SP)
286+
lxv vs63, 464(SP)
287+
288+
addi SP, SP, STACKSIZE
289+
blr
290+
291+
292+
EPILOGUE
293+
#endif

0 commit comments

Comments
 (0)