|
| 1 | +/*************************************************************************** |
| 2 | +Copyright (c) 2013-2019, The OpenBLAS Project |
| 3 | +All rights reserved. |
| 4 | +Redistribution and use in source and binary forms, with or without |
| 5 | +modification, are permitted provided that the following conditions are |
| 6 | +met: |
| 7 | +1. Redistributions of source code must retain the above copyright |
| 8 | +notice, this list of conditions and the following disclaimer. |
| 9 | +2. Redistributions in binary form must reproduce the above copyright |
| 10 | +notice, this list of conditions and the following disclaimer in |
| 11 | +the documentation and/or other materials provided with the |
| 12 | +distribution. |
| 13 | +3. Neither the name of the OpenBLAS project nor the names of |
| 14 | +its contributors may be used to endorse or promote products |
| 15 | +derived from this software without specific prior written permission. |
| 16 | +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" |
| 17 | +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
| 18 | +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
| 19 | +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE |
| 20 | +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
| 21 | +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR |
| 22 | +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
| 23 | +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, |
| 24 | +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE |
| 25 | +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 26 | +*****************************************************************************/ |
| 27 | + |
| 28 | +/************************************************************************************** |
| 29 | +* Abdelrauf(quickwritereader@gmail.com) |
| 30 | +* BLASTEST : OK |
| 31 | +* CTEST : OK |
| 32 | +* TEST : OK |
| 33 | +* LAPACK-TEST : OK |
| 34 | +**************************************************************************************/ |
| 35 | +#define ASSEMBLER |
| 36 | +#include "common.h" |
| 37 | +#include "def_vsx.h" |
| 38 | + |
| 39 | + |
| 40 | +#define LOAD ld |
| 41 | +#define STACKSIZE (512 ) |
| 42 | +#define FLINK_SAVE (STACKSIZE+16) /* 16($r12) */ |
| 43 | +#define M r3 |
| 44 | +#define N r4 |
| 45 | +#define K r5 |
| 46 | + |
| 47 | + |
| 48 | +#define A r8 |
| 49 | +#define B r9 |
| 50 | +#define C r10 |
| 51 | +#define LDC r6 |
| 52 | +#define OFFSET r7 |
| 53 | + |
| 54 | + |
| 55 | +#define alpha_r vs19 |
| 56 | +#define alpha_i vs20 |
| 57 | +#define save_permute_1 vs21 |
| 58 | +#define permute_mask vs22 |
| 59 | +#define o0 0 |
| 60 | + |
| 61 | + |
| 62 | +#define T1 r11 |
| 63 | +#define T2 r12 |
| 64 | +#define T3 r14 |
| 65 | +#define T4 r15 |
| 66 | +#define T5 r16 |
| 67 | +#define T6 r17 |
| 68 | +#define L r18 |
| 69 | +#define T7 r19 |
| 70 | +#define T8 r20 |
| 71 | +#define TEMP_REG r21 |
| 72 | +#define I r22 |
| 73 | +#define J r23 |
| 74 | +#define AO r24 |
| 75 | +#define BO r25 |
| 76 | +#define CO r26 |
| 77 | +#define T9 r27 |
| 78 | +#define T10 r28 |
| 79 | +#define PRE r29 |
| 80 | + |
| 81 | +#define T12 r30 |
| 82 | +#define T13 r31 |
| 83 | + |
| 84 | +#include "cgemm_macros_power9.S" |
| 85 | + |
| 86 | +.equ perm_const1, 0x0405060700010203 |
| 87 | +.equ perm_const2, 0x0c0d0e0f08090a0b |
| 88 | +.equ save_permute_12, 0x0c0d0e0f1c1d1e1f |
| 89 | +.equ save_permute_11, 0x0405060714151617 |
| 90 | + |
| 91 | + |
| 92 | + |
| 93 | +#ifndef NEEDPARAM |
| 94 | + |
| 95 | + PROLOGUE |
| 96 | + PROFCODE |
| 97 | + |
| 98 | + |
| 99 | + addi SP, SP, -STACKSIZE |
| 100 | + mflr r0 |
| 101 | + |
| 102 | + |
| 103 | + stfd f14, 0(SP) |
| 104 | + stfd f15, 8(SP) |
| 105 | + stfd f16, 16(SP) |
| 106 | + stfd f17, 24(SP) |
| 107 | + |
| 108 | + stfd f18, 32(SP) |
| 109 | + stfd f19, 40(SP) |
| 110 | + stfd f20, 48(SP) |
| 111 | + stfd f21, 56(SP) |
| 112 | + |
| 113 | + stfd f22, 64(SP) |
| 114 | + stfd f23, 72(SP) |
| 115 | + stfd f24, 80(SP) |
| 116 | + stfd f25, 88(SP) |
| 117 | + |
| 118 | + stfd f26, 96(SP) |
| 119 | + stfd f27, 104(SP) |
| 120 | + stfd f28, 112(SP) |
| 121 | + stfd f29, 120(SP) |
| 122 | + |
| 123 | + stfd f30, 128(SP) |
| 124 | + stfd f31, 136(SP) |
| 125 | + |
| 126 | + |
| 127 | + std r31, 144(SP) |
| 128 | + std r30, 152(SP) |
| 129 | + std r29, 160(SP) |
| 130 | + std r28, 168(SP) |
| 131 | + std r27, 176(SP) |
| 132 | + std r26, 184(SP) |
| 133 | + std r25, 192(SP) |
| 134 | + std r24, 200(SP) |
| 135 | + std r23, 208(SP) |
| 136 | + std r22, 216(SP) |
| 137 | + std r21, 224(SP) |
| 138 | + std r20, 232(SP) |
| 139 | + std r19, 240(SP) |
| 140 | + std r18, 248(SP) |
| 141 | + std r17, 256(SP) |
| 142 | + std r16, 264(SP) |
| 143 | + std r15, 272(SP) |
| 144 | + std r14, 280(SP) |
| 145 | + |
| 146 | + |
| 147 | + stxv vs52, 288(SP) |
| 148 | + stxv vs53, 304(SP) |
| 149 | + stxv vs54, 320(SP) |
| 150 | + stxv vs55, 336(SP) |
| 151 | + stxv vs56, 352(SP) |
| 152 | + stxv vs57, 368(SP) |
| 153 | + stxv vs58, 384(SP) |
| 154 | + stxv vs59, 400(SP) |
| 155 | + stxv vs60, 416(SP) |
| 156 | + stxv vs61, 432(SP) |
| 157 | + stxv vs62, 448(SP) |
| 158 | + stxv vs63, 464(SP) |
| 159 | + std r0, FLINK_SAVE(SP) |
| 160 | + |
| 161 | + |
| 162 | + |
| 163 | + ld LDC, FRAMESLOT(0) + STACKSIZE(SP) |
| 164 | + |
| 165 | + |
| 166 | + |
| 167 | +#ifdef TRMMKERNEL |
| 168 | + ld OFFSET, FRAMESLOT(1) + STACKSIZE(SP) |
| 169 | +#endif |
| 170 | + slwi LDC, LDC, ZBASE_SHIFT |
| 171 | + |
| 172 | + |
| 173 | + |
| 174 | + /*alpha is stored in f1. convert to single and splat*/ |
| 175 | + xscvdpspn alpha_r,vs1 |
| 176 | + xscvdpspn alpha_i,vs2 |
| 177 | + xxspltw alpha_r,alpha_r,0 |
| 178 | + xxspltw alpha_i,alpha_i,0 |
| 179 | +/*load reverse permute mask for big endian |
| 180 | + uint128 = 0xc0d0e0f08090a0b0405060700010203 |
| 181 | +*/ |
| 182 | + |
| 183 | + lis T2, perm_const2@highest |
| 184 | + lis T1, perm_const1@highest |
| 185 | + lis T3, save_permute_12@highest |
| 186 | + lis T4, save_permute_11@highest |
| 187 | + |
| 188 | + |
| 189 | + ori T2, T2, perm_const2@higher |
| 190 | + ori T1, T1, perm_const1@higher |
| 191 | + ori T3, T3, save_permute_12@higher |
| 192 | + ori T4, T4, save_permute_11@higher |
| 193 | + |
| 194 | + |
| 195 | + rldicr T2, T2, 32, 31 |
| 196 | + rldicr T1, T1, 32, 31 |
| 197 | + rldicr T3, T3, 32, 31 |
| 198 | + rldicr T4, T4, 32, 31 |
| 199 | + |
| 200 | + oris T2, T2, perm_const2@h |
| 201 | + oris T1, T1, perm_const1@h |
| 202 | + oris T3, T3, save_permute_12@h |
| 203 | + oris T4, T4, save_permute_11@h |
| 204 | + |
| 205 | + |
| 206 | + ori T2, T2, perm_const2@l |
| 207 | + ori T1, T1, perm_const1@l |
| 208 | + ori T3, T3, save_permute_12@l |
| 209 | + ori T4, T4, save_permute_11@l |
| 210 | + |
| 211 | + |
| 212 | + li r0,0 |
| 213 | + li PRE,512 |
| 214 | + |
| 215 | +#if defined(CC) || defined(CR) || defined(RC) || defined(RR) |
| 216 | +/*negate for this case as we will use addition -1*(a+b) */ |
| 217 | + xvnegsp alpha_r,alpha_r |
| 218 | + xvnegsp alpha_i,alpha_i |
| 219 | +#endif |
| 220 | + |
| 221 | + mtvsrdd permute_mask,T2,T1 |
| 222 | + mtvsrdd save_permute_1,T3,T4 |
| 223 | + |
| 224 | + /*mask is reverse permute so we have to make it inner permute */ |
| 225 | + xxpermdi permute_mask, permute_mask, permute_mask,2 |
| 226 | + |
| 227 | +#include "cgemm_logic_power9.S" |
| 228 | + |
| 229 | +.L999: |
| 230 | + lfd f14, 0(SP) |
| 231 | + lfd f15, 8(SP) |
| 232 | + lfd f16, 16(SP) |
| 233 | + lfd f17, 24(SP) |
| 234 | + |
| 235 | + lfd f18, 32(SP) |
| 236 | + lfd f19, 40(SP) |
| 237 | + lfd f20, 48(SP) |
| 238 | + lfd f21, 56(SP) |
| 239 | + |
| 240 | + lfd f22, 64(SP) |
| 241 | + lfd f23, 72(SP) |
| 242 | + lfd f24, 80(SP) |
| 243 | + lfd f25, 88(SP) |
| 244 | + |
| 245 | + lfd f26, 96(SP) |
| 246 | + lfd f27, 104(SP) |
| 247 | + lfd f28, 112(SP) |
| 248 | + lfd f29, 120(SP) |
| 249 | + |
| 250 | + lfd f30, 128(SP) |
| 251 | + lfd f31, 136(SP) |
| 252 | + |
| 253 | + ld r31, 144(SP) |
| 254 | + ld r30, 152(SP) |
| 255 | + ld r29, 160(SP) |
| 256 | + ld r28, 168(SP) |
| 257 | + ld r27, 176(SP) |
| 258 | + ld r26, 184(SP) |
| 259 | + ld r25, 192(SP) |
| 260 | + ld r24, 200(SP) |
| 261 | + ld r23, 208(SP) |
| 262 | + ld r22, 216(SP) |
| 263 | + ld r21, 224(SP) |
| 264 | + ld r20, 232(SP) |
| 265 | + ld r19, 240(SP) |
| 266 | + ld r18, 248(SP) |
| 267 | + ld r17, 256(SP) |
| 268 | + ld r16, 264(SP) |
| 269 | + ld r15, 272(SP) |
| 270 | + ld r14, 280(SP) |
| 271 | + |
| 272 | + ld r0, FLINK_SAVE(SP) |
| 273 | + |
| 274 | + lxv vs52, 288(SP) |
| 275 | + lxv vs53, 304(SP) |
| 276 | + lxv vs54, 320(SP) |
| 277 | + lxv vs55, 336(SP) |
| 278 | + lxv vs56, 352(SP) |
| 279 | + lxv vs57, 368(SP) |
| 280 | + lxv vs58, 384(SP) |
| 281 | + lxv vs59, 400(SP) |
| 282 | + mtlr r0 |
| 283 | + lxv vs60, 416(SP) |
| 284 | + lxv vs61, 432(SP) |
| 285 | + lxv vs62, 448(SP) |
| 286 | + lxv vs63, 464(SP) |
| 287 | + |
| 288 | + addi SP, SP, STACKSIZE |
| 289 | + blr |
| 290 | + |
| 291 | + |
| 292 | + EPILOGUE |
| 293 | +#endif |
0 commit comments