improved zgemm power9 based on power8

Author: quickwritereader

kernel/power/KERNEL.POWER9

@@ -38,7 +38,7 @@ CGEMMOTCOPYOBJ =  cgemm_otcopy.o
 CGEMMINCOPYOBJ =  cgemm_incopy.o
 CGEMMITCOPYOBJ =  cgemm_itcopy.o
 
-ZGEMMKERNEL    = zgemm_kernel_8x2_power8.S
+ZGEMMKERNEL    = zgemm_kernel_power9.S
 ZGEMMONCOPY    = ../generic/zgemm_ncopy_2.c
 ZGEMMOTCOPY    = ../generic/zgemm_tcopy_2.c
 ZGEMMINCOPY    = ../generic/zgemm_ncopy_8.c

kernel/power/sgemm_kernel_power9.S

@@ -168,7 +168,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 	/*alpha is stored in f1. convert to single and splat*/
-  xscvdpspn alpha_r,vs1 
+    xscvdpspn alpha_r,vs1 
 	xxspltw   alpha_r,alpha_r,0
 
 

kernel/power/sgemm_logic_power9.S

@@ -53,39 +53,39 @@ LSGEMM_L8x16_BEGIN:
 LSGEMM_L8x16_LOOP_START:
 
 	LOAD8x16_0  /*we already zeroed */
-    ##OffsetA=64 OffsetB=32
-    addi AO,AO,2112
-    addi BO,BO,32  
+    /*##OffsetA=64 OffsetB=32
+    #addi AO,AO,2112
+    #addi BO,BO,32  */
 
 	mtctr		L
 
 	MY_ALIGN
 
 LSGEMM_L8x16_LOOP:
 
-    KERNEL8x16_I1_L4_2  -2048,0, 0,0
-    KERNEL8x16_I1_L4_2  -2048,0, 1,0
-    KERNEL8x16_I1_L4_2  -2048,0, 2,0
-    KERNEL8x16_I1_L4_2  -2048,0, 3,0
-    KERNEL8x16_I1_L4_2  -2048,0, 4,0
-    KERNEL8x16_I1_L4_2  -2048,0, 5,0        
-    KERNEL8x16_I1_L4_2  -2048,0, 6,0
-    KERNEL8x16_I1_L4_2  -2048,0, 7,0  
-    KERNEL8x16_I1_L4_2  -2048,0, 8,0      
-    KERNEL8x16_I1_L4_2  -2048,0, 9,0
-    KERNEL8x16_I1_L4_2  -2048,0, 10,0
-    KERNEL8x16_I1_L4_2  -2048,0, 11,0
-    KERNEL8x16_I1_L4_2  -2048,0, 12,0
-    KERNEL8x16_I1_L4_2  -2048,0, 13,0    
-    KERNEL8x16_I1_L4_2  -2048,0, 14,0    
-    KERNEL8x16_I1_L4_2  -2048,0, 15,1  	
+    KERNEL8x16_I1_L4_2  64,32, 0,0
+    KERNEL8x16_I1_L4_2  64,32, 1,0
+    KERNEL8x16_I1_L4_2  64,32, 2,0
+    KERNEL8x16_I1_L4_2  64,32, 3,0
+    KERNEL8x16_I1_L4_2  64,32, 4,0
+    KERNEL8x16_I1_L4_2  64,32, 5,0        
+    KERNEL8x16_I1_L4_2  64,32, 6,0
+    KERNEL8x16_I1_L4_2  64,32, 7,0  
+    KERNEL8x16_I1_L4_2  64,32, 8,0      
+    KERNEL8x16_I1_L4_2  64,32, 9,0
+    KERNEL8x16_I1_L4_2  64,32, 10,0
+    KERNEL8x16_I1_L4_2  64,32, 11,0
+    KERNEL8x16_I1_L4_2  64,32, 12,0
+    KERNEL8x16_I1_L4_2  64,32, 13,0    
+    KERNEL8x16_I1_L4_2  64,32, 14,0    
+    KERNEL8x16_I1_L4_2  64,32, 15,1  	
 
 	bdnz		LSGEMM_L8x16_LOOP
 
 	MY_ALIGN
 LSGEMM_L8x16_LOOP_END:
 
-    END8x16 0, AO, BO, -2048, 0    
+    END8x16 0, AO, BO, 64, 32    
 
 	b		LSGEMM_L8x16_SUB1 
 	MY_ALIGN

kernel/power/zgemm_kernel_power9.S

@@ -0,0 +1,257 @@
+/***************************************************************************
+Copyright (c) 2013-2019, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+#define ASSEMBLER
+#include "common.h"
+#include "def_vsx.h"
+
+#define LOAD	ld
+ 
+#define STACKSIZE 32192
+
+#define FZERO	312+192(SP)
+ 
+
+#define	M	r3
+#define	N	r4
+#define	K	r5
+
+ 
+#define A	r8
+#define	B	r9
+#define	C	r10
+#define	LDC	r6
+#define OFFSET	r7
+ 
+ 
+
+#define o0	0
+#define alpha_r vs30
+#define alpha_i vs31
+
+#define VECSAVE r11
+
+#define FRAMEPOINTER r12
+
+#define BBUFFER r14
+
+#define L	r15
+#define ALPHA	r16
+#define T5	r17
+#define T2	r19
+#define BBO	r20
+#define	o8	r21
+#define	I	r22
+#define J	r23
+#define AO	r24
+#define	BO	r25
+#define	CO	r26
+#define o16	r27
+#define	T3	r28
+#define T4	r29
+
+#define PRE	r30
+#define T1  	r31
+
+#ifndef NEEDPARAM
+
+	PROLOGUE
+	PROFCODE
+
+	mr      FRAMEPOINTER, SP
+        addi    SP, SP, -STACKSIZE
+        addi    SP, SP, -STACKSIZE
+        addi    SP, SP, -STACKSIZE
+        addi    SP, SP, -STACKSIZE
+        li      r0, 0
+
+	stfd	f14,    0(SP)
+	stfd	f15,    8(SP)
+	stfd	f16,   16(SP)
+	stfd	f17,   24(SP)
+
+	stfd	f18,   32(SP)
+	stfd	f19,   40(SP)
+	stfd	f20,   48(SP)
+	stfd	f21,   56(SP)
+
+	stfd	f22,   64(SP)
+	stfd	f23,   72(SP)
+	stfd	f24,   80(SP)
+	stfd	f25,   88(SP)
+
+	stfd	f26,   96(SP)
+	stfd	f27,  104(SP)
+	stfd	f28,  112(SP)
+	stfd	f29,  120(SP)
+
+	stfd	f30,  128(SP)
+	stfd	f31,  136(SP)
+
+ 
+	std	r31,  144(SP)
+	std	r30,  152(SP)
+	std	r29,  160(SP)
+	std	r28,  168(SP)
+	std	r27,  176(SP)
+	std	r26,  184(SP)
+	std	r25,  192(SP)
+	std	r24,  200(SP)
+	std	r23,  208(SP)
+	std	r22,  216(SP)
+	std	r21,  224(SP)
+	std	r20,  232(SP)
+	std	r19,  240(SP)
+	std	r18,  248(SP)
+	std	r17,  256(SP)
+	std	r16,  264(SP)
+	std	r15,  272(SP)
+	std	r14,  280(SP)
+ 
+ 
+    stxv    v20,  288(SP)
+    stxv    v21,  304(SP)
+    stxv    v22,  320(SP)
+    stxv    v23,  336(SP)
+    stxv    v24,  352(SP)
+    stxv    v25,  368(SP)
+    stxv    v26,  384(SP)
+    stxv    v27,  400(SP)
+    stxv    v28,  416(SP)
+    stxv    v29,  432(SP)
+    stxv    v30,  448(SP)
+    stxv    v31,  464(SP)
+
+ 
+	stw	r0,  FZERO
+
+#ifdef linux
+	ld	LDC, FRAMESLOT(0) + 0(FRAMEPOINTER)
+#endif
+
+
+#ifdef TRMMKERNEL
+#if defined(linux) && defined(__64BIT__)
+	ld	OFFSET,  FRAMESLOT(1) + 0(FRAMEPOINTER)
+#endif 
+#endif
+
+
+#include "zgemm_macros_power9.S"
+
+	cmpwi	cr0, M, 0
+	ble	L999
+	cmpwi	cr0, N, 0
+	ble	L999
+	cmpwi	cr0, K, 0
+	ble	L999
+
+	slwi	LDC, LDC, ZBASE_SHIFT
+	li	PRE,  512
+	li	o8  , 8
+	li	o16 , 16 
+
+    addi    BBUFFER, SP, 512+4096
+    li      T1, -4096
+    and     BBUFFER, BBUFFER, T1
+
+ 
+	addi	ALPHA, SP, 296+192
+ 
+    xxlor  alpha_r,vs1,vs1  /*copy from register f1 */
+    xxlor  alpha_i,vs2,vs2  /*copy from register f2 */
+
+	.align 4
+
+#include "zgemm_logic_power9.S"
+
+L999:
+	addi	r3, 0, 0
+
+	lfd	f14,    0(SP)
+	lfd	f15,    8(SP)
+	lfd	f16,   16(SP)
+	lfd	f17,   24(SP)
+
+	lfd	f18,   32(SP)
+	lfd	f19,   40(SP)
+	lfd	f20,   48(SP)
+	lfd	f21,   56(SP)
+
+	lfd	f22,   64(SP)
+	lfd	f23,   72(SP)
+	lfd	f24,   80(SP)
+	lfd	f25,   88(SP)
+
+	lfd	f26,   96(SP)
+	lfd	f27,  104(SP)
+	lfd	f28,  112(SP)
+	lfd	f29,  120(SP)
+
+	lfd	f30,  128(SP)
+	lfd	f31,  136(SP)
+
+ 
+	ld	r31,  144(SP)
+	ld	r30,  152(SP)
+	ld	r29,  160(SP)
+	ld	r28,  168(SP)
+	ld	r27,  176(SP)
+	ld	r26,  184(SP)
+	ld	r25,  192(SP)
+	ld	r24,  200(SP)
+	ld	r23,  208(SP)
+	ld	r22,  216(SP)
+	ld	r21,  224(SP)
+	ld	r20,  232(SP)
+	ld	r19,  240(SP)
+	ld	r18,  248(SP)
+	ld	r17,  256(SP)
+	ld	r16,  264(SP)
+	ld	r15,  272(SP)
+	ld	r14,  280(SP)
+ 
+	lxv    v20,  288(SP)
+	lxv    v21,  304(SP)
+	lxv    v22,  320(SP)
+	lxv    v23,  336(SP)
+	lxv    v24,  352(SP)
+	lxv    v25,  368(SP)
+	lxv    v26,  384(SP)
+	lxv    v27,  400(SP)
+	lxv    v28,  416(SP)
+	lxv    v29,  432(SP)
+	lxv    v30,  448(SP)
+	lxv    v31,  464(SP)
+
+	addi	SP, SP, STACKSIZE
+	addi	SP, SP, STACKSIZE
+	addi	SP, SP, STACKSIZE
+	addi	SP, SP, STACKSIZE
+	blr
+
+	EPILOGUE
+#endif