Skip to content

Commit 6c85cb1

Browse files
martin-frbgmartin-frbg
authored
Merge pull request #2352 from wjc404/develop
AVX2 ZGEMM3M kernel
2 parents 995768b + a0f0a80 commit 6c85cb1

File tree

5 files changed

+235
-11
lines changed

5 files changed

+235
-11
lines changed

‎CONTRIBUTORS.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -178,4 +178,4 @@ In chronological order:
178178
* [2019-11-06] optimize AVX512 SGEMM
179179
* [2019-11-12] AVX512 CGEMM & ZGEMM kernels
180180
* [2019-12-23] optimize AVX2 CGEMM and ZGEMM
181-
* [2019-12-27] AVX2 CGEMM3M kernel
181+
* [2019-12-30] AVX2 CGEMM3M & ZGEMM3M kernels

‎kernel/x86_64/KERNEL.HASWELL

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,5 +98,5 @@ ZTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c
9898
ZTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c
9999

100100
CGEMM3MKERNEL = cgemm3m_kernel_8x4_haswell.c
101-
ZGEMM3MKERNEL = zgemm3m_kernel_2x8_nehalem.S
101+
ZGEMM3MKERNEL = zgemm3m_kernel_4x4_haswell.c
102102

‎kernel/x86_64/KERNEL.ZEN

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,5 +95,5 @@ ZTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c
9595
ZTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c
9696

9797
CGEMM3MKERNEL = cgemm3m_kernel_8x4_haswell.c
98-
ZGEMM3MKERNEL = zgemm3m_kernel_2x8_nehalem.S
98+
ZGEMM3MKERNEL = zgemm3m_kernel_4x4_haswell.c
9999

‎kernel/x86_64/zgemm3m_kernel_4x4_haswell.c

Lines changed: 224 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,224 @@
1+
/* %0 = "+r"(a_pointer), %1 = "+r"(b_pointer), %2 = "+r"(c_pointer), %3 = "+r"(ldc_in_bytes), %4 for k_count, %5 for c_store */
2+
/* r12 = k << 5(const), r13 = k(const), r14 = b_head_pos(const), r15 = tmp */
3+
4+
#include "common.h"
5+
#include <stdint.h>
6+
7+
//recommended settings: GEMM_Q=256, GEMM_P=256
8+
9+
/* m = 4 *//* ymm0 for alpha, ymm1-ymm3 for temporary use, ymm4-ymm15 for accumulators */
10+
#define KERNEL_k1m4n1 \
11+
"vmovupd (%0),%%ymm1; addq $32,%0;"\
12+
"vbroadcastsd (%1),%%ymm2; vfmadd231pd %%ymm1,%%ymm2,%%ymm4;"\
13+
"addq $8,%1;"
14+
#define KERNEL_h_k1m4n2 \
15+
"vmovddup (%0),%%ymm1; vmovddup 8(%0),%%ymm2; addq $32,%0;"\
16+
"vbroadcastf128 (%1),%%ymm3; vfmadd231pd %%ymm1,%%ymm3,%%ymm4; vfmadd231pd %%ymm2,%%ymm3,%%ymm5;"
17+
#define KERNEL_k1m4n2 KERNEL_h_k1m4n2 "addq $16,%1;"
18+
#define KERNEL_h_k1m4n4 \
19+
KERNEL_h_k1m4n2 "vbroadcastf128 16(%1),%%ymm3; vfmadd231pd %%ymm1,%%ymm3,%%ymm6; vfmadd231pd %%ymm2,%%ymm3,%%ymm7;"
20+
#define KERNEL_k1m4n4 KERNEL_h_k1m4n4 "addq $32,%1;"
21+
#define unit_kernel_k1m4n4(c1,c2,c3,c4,off1,off2,...) \
22+
"vbroadcastf128 "#off1"("#__VA_ARGS__"),%%ymm3; vfmadd231pd %%ymm1,%%ymm3,"#c1"; vfmadd231pd %%ymm2,%%ymm3,"#c2";"\
23+
"vbroadcastf128 "#off2"("#__VA_ARGS__"),%%ymm3; vfmadd231pd %%ymm1,%%ymm3,"#c3"; vfmadd231pd %%ymm2,%%ymm3,"#c4";"
24+
#define KERNEL_h_k1m4n8 KERNEL_h_k1m4n4 unit_kernel_k1m4n4(%%ymm8,%%ymm9,%%ymm10,%%ymm11,0,16,%1,%%r12,1)
25+
#define KERNEL_k1m4n8 KERNEL_h_k1m4n8 "addq $32,%1;"
26+
#define KERNEL_h_k1m4n12 KERNEL_h_k1m4n8 unit_kernel_k1m4n4(%%ymm12,%%ymm13,%%ymm14,%%ymm15,0,16,%1,%%r12,2)
27+
#define KERNEL_k1m4n12 KERNEL_h_k1m4n12 "addq $32,%1;"
28+
#define KERNEL_k2m4n1 KERNEL_k1m4n1 KERNEL_k1m4n1
29+
#define KERNEL_k2m4n2 KERNEL_k1m4n2 KERNEL_k1m4n2
30+
#define KERNEL_k2m4n4 KERNEL_k1m4n4 KERNEL_k1m4n4
31+
#define KERNEL_k2m4n8 KERNEL_k1m4n8 KERNEL_k1m4n8
32+
#define KERNEL_k2m4n12 \
33+
"vmovddup (%0),%%ymm1; vmovddup 8(%0),%%ymm2;"\
34+
unit_kernel_k1m4n4(%%ymm4,%%ymm5,%%ymm6,%%ymm7,0,16,%1)\
35+
unit_kernel_k1m4n4(%%ymm8,%%ymm9,%%ymm10,%%ymm11,0,16,%1,%%r12,1)\
36+
unit_kernel_k1m4n4(%%ymm12,%%ymm13,%%ymm14,%%ymm15,0,16,%1,%%r12,2)\
37+
"vmovddup 32(%0),%%ymm1; vmovddup 40(%0),%%ymm2; prefetcht0 512(%0); addq $64,%0;"\
38+
unit_kernel_k1m4n4(%%ymm4,%%ymm5,%%ymm6,%%ymm7,32,48,%1)\
39+
unit_kernel_k1m4n4(%%ymm8,%%ymm9,%%ymm10,%%ymm11,32,48,%1,%%r12,1)\
40+
unit_kernel_k1m4n4(%%ymm12,%%ymm13,%%ymm14,%%ymm15,32,48,%1,%%r12,2) "addq $64,%1;"
41+
#define INIT_m4n1 "vpxor %%ymm4,%%ymm4,%%ymm4;"
42+
#define INIT_m4n2 INIT_m4n1 "vpxor %%ymm5,%%ymm5,%%ymm5;"
43+
#define INIT_m4n4 INIT_m4n2 "vpxor %%ymm6,%%ymm6,%%ymm6;vpxor %%ymm7,%%ymm7,%%ymm7;"
44+
#define unit_init_m4n4(c1,c2,c3,c4) \
45+
"vpxor "#c1","#c1","#c1";vpxor "#c2","#c2","#c2";vpxor "#c3","#c3","#c3";vpxor "#c4","#c4","#c4";"
46+
#define INIT_m4n8 INIT_m4n4 unit_init_m4n4(%%ymm8,%%ymm9,%%ymm10,%%ymm11)
47+
#define INIT_m4n12 INIT_m4n8 unit_init_m4n4(%%ymm12,%%ymm13,%%ymm14,%%ymm15)
48+
#define SAVE_h_m4n1 \
49+
"vpermpd $216,%%ymm4,%%ymm3; vunpcklpd %%ymm3,%%ymm3,%%ymm1; vunpckhpd %%ymm3,%%ymm3,%%ymm2;"\
50+
"vfmadd213pd (%2),%%ymm0,%%ymm1; vfmadd213pd 32(%2),%%ymm0,%%ymm2; vmovupd %%ymm1,(%2); vmovupd %%ymm2,32(%2);"
51+
#define unit_save_m4n2(c1,c2) \
52+
"vperm2f128 $2,"#c1","#c2",%%ymm2; vperm2f128 $19,"#c1","#c2","#c2"; vmovapd %%ymm2,"#c1";"\
53+
"vunpcklpd "#c1","#c1",%%ymm2; vunpcklpd "#c2","#c2",%%ymm3;"\
54+
"vfmadd213pd (%5),%%ymm0,%%ymm2; vfmadd213pd 32(%5),%%ymm0,%%ymm3; vmovupd %%ymm2,(%5); vmovupd %%ymm3,32(%5);"\
55+
"vunpckhpd "#c1","#c1",%%ymm2; vunpckhpd "#c2","#c2",%%ymm3;"\
56+
"vfmadd213pd (%5,%3,1),%%ymm0,%%ymm2; vfmadd213pd 32(%5,%3,1),%%ymm0,%%ymm3; vmovupd %%ymm2,(%5,%3,1); vmovupd %%ymm3,32(%5,%3,1);"\
57+
"leaq (%5,%3,2),%5;"
58+
#define SAVE_h_m4n2 "movq %2,%5;" unit_save_m4n2(%%ymm4,%%ymm5)
59+
#define SAVE_h_m4n4 SAVE_h_m4n2 unit_save_m4n2(%%ymm6,%%ymm7)
60+
#define SAVE_h_m4n8 SAVE_h_m4n4 unit_save_m4n2(%%ymm8,%%ymm9) unit_save_m4n2(%%ymm10,%%ymm11)
61+
#define SAVE_h_m4n12 SAVE_h_m4n8 unit_save_m4n2(%%ymm12,%%ymm13) unit_save_m4n2(%%ymm14,%%ymm15)
62+
#define SAVE_m4(ndim) SAVE_h_m4n##ndim "addq $64,%2;"
63+
#define COMPUTE_m4(ndim) \
64+
INIT_m4n##ndim\
65+
"movq %%r13,%4; movq %%r14,%1; movq %2,%5; xorq %%r15,%%r15;"\
66+
"cmpq $24,%4; jb "#ndim"004042f;"\
67+
#ndim"004041:\n\t"\
68+
"cmpq $126,%%r15; movq $126,%%r15; cmoveq %3,%%r15;"\
69+
KERNEL_k2m4n##ndim KERNEL_k2m4n##ndim\
70+
"prefetcht1 (%5); subq $63,%5;"\
71+
KERNEL_k2m4n##ndim KERNEL_k2m4n##ndim\
72+
"addq %%r15,%5; prefetcht1 (%8); addq $32,%8;"\
73+
"subq $8,%4; cmpq $16,%4; jnb "#ndim"004041b;"\
74+
"movq %2,%5;"\
75+
#ndim"004042:\n\t"\
76+
"testq %4,%4; jz "#ndim"004043f;"\
77+
"prefetcht0 (%5); prefetcht0 63(%5);"\
78+
KERNEL_k1m4n##ndim\
79+
"prefetcht0 (%5,%3,4); prefetcht0 63(%5,%3,4); addq %3,%5;"\
80+
"decq %4; jmp "#ndim"004042b;"\
81+
#ndim"004043:\n\t"\
82+
"prefetcht0 (%%r14); prefetcht0 64(%%r14);"\
83+
SAVE_m4(ndim)
84+
85+
/* m = 2 *//* vmm0 for alpha, vmm1-vmm3 for temporary use, vmm4-vmm9 for accumulators */
86+
#define KERNEL_k1m2n1 \
87+
"vmovupd (%0),%%xmm1; addq $16,%0;"\
88+
"vmovddup (%1),%%xmm2; vfmadd231pd %%xmm1,%%xmm2,%%xmm4;"\
89+
"addq $8,%1;"
90+
#define KERNEL_h_k1m2n2 \
91+
"vmovddup (%0),%%xmm1; vmovddup 8(%0),%%xmm2; addq $16,%0;"\
92+
"vmovupd (%1),%%xmm3; vfmadd231pd %%xmm1,%%xmm3,%%xmm4; vfmadd231pd %%xmm2,%%xmm3,%%xmm5;"
93+
#define KERNEL_k1m2n2 KERNEL_h_k1m2n2 "addq $16,%1;"
94+
#define unit_kernel_k1m2n4(c1,c2,...) \
95+
"vmovupd ("#__VA_ARGS__"),%%ymm3; vfmadd231pd %%ymm1,%%ymm3,"#c1"; vfmadd231pd %%ymm2,%%ymm3,"#c2";"
96+
#define KERNEL_h_k1m2n4 \
97+
"vbroadcastsd (%0),%%ymm1; vbroadcastsd 8(%0),%%ymm2; addq $16,%0;"\
98+
unit_kernel_k1m2n4(%%ymm4,%%ymm5,%1)
99+
#define KERNEL_k1m2n4 KERNEL_h_k1m2n4 "addq $32,%1;"
100+
#define KERNEL_h_k1m2n8 KERNEL_h_k1m2n4 \
101+
unit_kernel_k1m2n4(%%ymm6,%%ymm7,%1,%%r12,1)
102+
#define KERNEL_k1m2n8 KERNEL_h_k1m2n8 "addq $32,%1;"
103+
#define KERNEL_h_k1m2n12 KERNEL_h_k1m2n8 \
104+
unit_kernel_k1m2n4(%%ymm8,%%ymm9,%1,%%r12,2)
105+
#define KERNEL_k1m2n12 KERNEL_h_k1m2n12 "addq $32,%1;"
106+
#define INIT_m2n1 "vpxor %%xmm4,%%xmm4,%%xmm4;"
107+
#define INIT_m2n2 INIT_m2n1 "vpxor %%xmm5,%%xmm5,%%xmm5;"
108+
#define unit_init_m2n4(c1,c2) "vpxor "#c1","#c1","#c1";vpxor "#c2","#c2","#c2";"
109+
#define INIT_m2n4 unit_init_m2n4(%%ymm4,%%ymm5)
110+
#define INIT_m2n8 INIT_m2n4 unit_init_m2n4(%%ymm6,%%ymm7)
111+
#define INIT_m2n12 INIT_m2n8 unit_init_m2n4(%%ymm8,%%ymm9)
112+
#define SAVE_h_m2n1 \
113+
"vinsertf128 $1,%%xmm4,%%ymm4,%%ymm4; vpermilpd $12,%%ymm4,%%ymm4; vfmadd213pd (%2),%%ymm0,%%ymm4; vmovupd %%ymm4,(%2);"
114+
#define SAVE_h_m2n2 \
115+
"vinsertf128 $1,%%xmm5,%%ymm4,%%ymm4; vunpcklpd %%ymm4,%%ymm4,%%ymm1; vunpckhpd %%ymm4,%%ymm4,%%ymm2;"\
116+
"vfmadd213pd (%2),%%ymm0,%%ymm1; vmovupd %%ymm1,(%2);"\
117+
"vfmadd213pd (%2,%3,1),%%ymm0,%%ymm2; vmovupd %%ymm2,(%2,%3,1);"
118+
#define unit_save_m2n4(c1,c2) \
119+
"vperm2f128 $2,"#c1","#c2",%%ymm1; vunpcklpd %%ymm1,%%ymm1,%%ymm2; vunpckhpd %%ymm1,%%ymm1,%%ymm3;"\
120+
"vfmadd213pd (%5),%%ymm0,%%ymm2; vfmadd213pd (%5,%3,1),%%ymm0,%%ymm3; vmovupd %%ymm2,(%5); vmovupd %%ymm3,(%5,%3,1); leaq (%5,%3,2),%5;"\
121+
"vperm2f128 $19,"#c1","#c2",%%ymm1; vunpcklpd %%ymm1,%%ymm1,%%ymm2; vunpckhpd %%ymm1,%%ymm1,%%ymm3;"\
122+
"vfmadd213pd (%5),%%ymm0,%%ymm2; vfmadd213pd (%5,%3,1),%%ymm0,%%ymm3; vmovupd %%ymm2,(%5); vmovupd %%ymm3,(%5,%3,1); leaq (%5,%3,2),%5;"
123+
#define SAVE_h_m2n4 "movq %2,%5;" unit_save_m2n4(%%ymm4,%%ymm5)
124+
#define SAVE_h_m2n8 SAVE_h_m2n4 unit_save_m2n4(%%ymm6,%%ymm7)
125+
#define SAVE_h_m2n12 SAVE_h_m2n8 unit_save_m2n4(%%ymm8,%%ymm9)
126+
#define SAVE_m2(ndim) SAVE_h_m2n##ndim "addq $32,%2;"
127+
#define COMPUTE_m2(ndim) \
128+
INIT_m2n##ndim\
129+
"movq %%r13,%4; movq %%r14,%1;"\
130+
#ndim"002022:\n\t"\
131+
"testq %4,%4; jz "#ndim"002023f;"\
132+
KERNEL_k1m2n##ndim\
133+
"decq %4; jmp "#ndim"002022b;"\
134+
#ndim"002023:\n\t"\
135+
SAVE_m2(ndim)
136+
137+
/* m = 1 *//* vmm0 for alpha, vmm1-vmm3 and vmm10-vmm15 for temporary use, vmm4-vmm6 for accumulators */
138+
#define KERNEL_k1m1n1 \
139+
"vmovsd (%0),%%xmm1; addq $8,%0;"\
140+
"vfmadd231sd (%1),%%xmm1,%%xmm4; addq $8,%1;"
141+
#define KERNEL_k1m1n2 \
142+
"vmovddup (%0),%%xmm1; addq $8,%0;"\
143+
"vfmadd231pd (%1),%%xmm1,%%xmm4; addq $16,%1;"
144+
#define unit_kernel_k1m1n4(c1,...) \
145+
"vmovupd ("#__VA_ARGS__"),%%ymm2; vfmadd231pd %%ymm1,%%ymm2,"#c1";"
146+
#define KERNEL_h_k1m1n4 \
147+
"vbroadcastsd (%0),%%ymm1; addq $8,%0;"\
148+
unit_kernel_k1m1n4(%%ymm4,%1)
149+
#define KERNEL_k1m1n4 KERNEL_h_k1m1n4 "addq $32,%1;"
150+
#define KERNEL_h_k1m1n8 KERNEL_h_k1m1n4 unit_kernel_k1m1n4(%%ymm5,%1,%%r12,1)
151+
#define KERNEL_k1m1n8 KERNEL_h_k1m1n8 "addq $32,%1;"
152+
#define KERNEL_h_k1m1n12 KERNEL_h_k1m1n8 unit_kernel_k1m1n4(%%ymm6,%1,%%r12,2)
153+
#define KERNEL_k1m1n12 KERNEL_h_k1m1n12 "addq $32,%1;"
154+
#define INIT_m1n1 INIT_m2n1
155+
#define INIT_m1n2 INIT_m2n1
156+
#define INIT_m1n4 "vpxor %%ymm4,%%ymm4,%%ymm4;"
157+
#define INIT_m1n8 INIT_m1n4 "vpxor %%ymm5,%%ymm5,%%ymm5;"
158+
#define INIT_m1n12 INIT_m1n8 "vpxor %%ymm6,%%ymm6,%%ymm6;"
159+
#define SAVE_h_m1n1 \
160+
"vmovddup %%xmm4,%%xmm4; vfmadd213pd (%2),%%xmm0,%%xmm4; vmovupd %%xmm4,(%2);"
161+
#define SAVE_h_m1n2 \
162+
"vunpcklpd %%xmm4,%%xmm4,%%xmm1; vunpckhpd %%xmm4,%%xmm4,%%xmm2;"\
163+
"vfmadd213pd (%2),%%xmm0,%%xmm1; vmovupd %%xmm1,(%2);"\
164+
"vfmadd213pd (%2,%3,1),%%xmm0,%%xmm2; vmovupd %%xmm2,(%2,%3,1);"
165+
#define unit_save_m1n4(c1) \
166+
"vunpcklpd "#c1","#c1",%%ymm1; vunpckhpd "#c1","#c1",%%ymm2;"\
167+
"vmovupd (%5),%%xmm3; vinsertf128 $1,(%5,%3,2),%%ymm3,%%ymm3;"\
168+
"vfmadd213pd %%ymm3,%%ymm0,%%ymm1; vmovupd %%xmm1,(%5); vextractf128 $1,%%ymm1,(%5,%3,2); addq %3,%5;"\
169+
"vmovupd (%5),%%xmm3; vinsertf128 $1,(%5,%3,2),%%ymm3,%%ymm3;"\
170+
"vfmadd213pd %%ymm3,%%ymm0,%%ymm2; vmovupd %%xmm2,(%5); vextractf128 $1,%%ymm2,(%5,%3,2); addq %3,%5; leaq (%5,%3,2),%5;"
171+
#define SAVE_h_m1n4 "movq %2,%5;" unit_save_m1n4(%%ymm4)
172+
#define SAVE_h_m1n8 SAVE_h_m1n4 unit_save_m1n4(%%ymm5)
173+
#define SAVE_h_m1n12 SAVE_h_m1n8 unit_save_m1n4(%%ymm6)
174+
#define SAVE_m1(ndim) SAVE_h_m1n##ndim "addq $16,%2;"
175+
#define COMPUTE_m1(ndim) \
176+
INIT_m1n##ndim\
177+
"movq %%r13,%4; movq %%r14,%1;"\
178+
#ndim"001011:\n\t"\
179+
"testq %4,%4; jz "#ndim"001012f;"\
180+
KERNEL_k1m1n##ndim\
181+
"decq %4; jmp "#ndim"001011b;"\
182+
#ndim"001012:\n\t"\
183+
SAVE_m1(ndim)
184+
185+
#define COMPUTE(ndim) {\
186+
next_b = b_pointer + ndim * K;\
187+
__asm__ __volatile__(\
188+
"vbroadcastf128 (%6),%%ymm0;"\
189+
"movq %4,%%r13; movq %4,%%r12; salq $5,%%r12; movq %1,%%r14; movq %7,%%r11;"\
190+
"cmpq $4,%7;jb 33101"#ndim"f;"\
191+
"33109"#ndim":\n\t"\
192+
COMPUTE_m4(ndim)\
193+
"subq $4,%7;cmpq $4,%7;jnb 33109"#ndim"b;"\
194+
"33101"#ndim":\n\t"\
195+
"cmpq $2,%7;jb 33104"#ndim"f;"\
196+
COMPUTE_m2(ndim)\
197+
"subq $2,%7;"\
198+
"33104"#ndim":\n\t"\
199+
"testq %7,%7;jz 33105"#ndim"f;"\
200+
COMPUTE_m1(ndim)\
201+
"33105"#ndim":\n\t"\
202+
"movq %%r13,%4; movq %%r14,%1; movq %%r11,%7;"\
203+
:"+r"(a_pointer),"+r"(b_pointer),"+r"(c_pointer),"+r"(ldc_in_bytes),"+r"(K),"+r"(ctemp),"+r"(const_val),"+r"(M),"+r"(next_b)\
204+
::"r11","r12","r13","r14","r15","xmm0","xmm1","xmm2","xmm3","xmm4","xmm5","xmm6","xmm7","xmm8","xmm9","xmm10","xmm11","xmm12","xmm13","xmm14",\
205+
"xmm15","cc","memory");\
206+
a_pointer -= M * K; b_pointer += ndim * K; c_pointer += 2*(LDC * ndim - M);\
207+
}
208+
int __attribute__ ((noinline))
209+
CNAME(BLASLONG m, BLASLONG n, BLASLONG k, double alphar, double alphai, double * __restrict__ A, double * __restrict__ B, double * __restrict__ C, BLASLONG LDC)
210+
{
211+
if(m==0||n==0||k==0) return 0;
212+
int64_t ldc_in_bytes = (int64_t)LDC * sizeof(double) * 2;
213+
double constval[2]; constval[0] = alphar; constval[1] = alphai;
214+
double *const_val=constval;
215+
int64_t M = (int64_t)m, K = (int64_t)k;
216+
BLASLONG n_count = n;
217+
double *a_pointer = A,*b_pointer = B,*c_pointer = C,*ctemp = C,*next_b = B;
218+
for(;n_count>11;n_count-=12) COMPUTE(12)
219+
for(;n_count>7;n_count-=8) COMPUTE(8)
220+
for(;n_count>3;n_count-=4) COMPUTE(4)
221+
for(;n_count>1;n_count-=2) COMPUTE(2)
222+
if(n_count>0) COMPUTE(1)
223+
return 0;
224+
}

‎param.h

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -695,14 +695,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
695695

696696
#define CGEMM3M_DEFAULT_UNROLL_N 4
697697
#define CGEMM3M_DEFAULT_UNROLL_M 8
698-
#define ZGEMM3M_DEFAULT_UNROLL_N 8
699-
#define ZGEMM3M_DEFAULT_UNROLL_M 2
698+
#define ZGEMM3M_DEFAULT_UNROLL_N 4
699+
#define ZGEMM3M_DEFAULT_UNROLL_M 4
700700

701701
#define CGEMM3M_DEFAULT_P 320
702-
#define ZGEMM3M_DEFAULT_P 224
702+
#define ZGEMM3M_DEFAULT_P 256
703703
#define XGEMM3M_DEFAULT_P 112
704704
#define CGEMM3M_DEFAULT_Q 320
705-
#define ZGEMM3M_DEFAULT_Q 224
705+
#define ZGEMM3M_DEFAULT_Q 256
706706
#define XGEMM3M_DEFAULT_Q 224
707707
#define CGEMM3M_DEFAULT_R 12288
708708
#define ZGEMM3M_DEFAULT_R 12288
@@ -1598,14 +1598,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
15981598

15991599
#define CGEMM3M_DEFAULT_UNROLL_N 4
16001600
#define CGEMM3M_DEFAULT_UNROLL_M 8
1601-
#define ZGEMM3M_DEFAULT_UNROLL_N 8
1602-
#define ZGEMM3M_DEFAULT_UNROLL_M 2
1601+
#define ZGEMM3M_DEFAULT_UNROLL_N 4
1602+
#define ZGEMM3M_DEFAULT_UNROLL_M 4
16031603

16041604
#define CGEMM3M_DEFAULT_P 320
1605-
#define ZGEMM3M_DEFAULT_P 224
1605+
#define ZGEMM3M_DEFAULT_P 256
16061606
#define XGEMM3M_DEFAULT_P 112
16071607
#define CGEMM3M_DEFAULT_Q 320
1608-
#define ZGEMM3M_DEFAULT_Q 224
1608+
#define ZGEMM3M_DEFAULT_Q 256
16091609
#define XGEMM3M_DEFAULT_Q 224
16101610
#define CGEMM3M_DEFAULT_R 12288
16111611
#define ZGEMM3M_DEFAULT_R 12288

0 commit comments

Comments
 (0)