metal : tune soft_max number of threads (whisper/0)

ggerganov · ggerganov · commit f308ea705974 · 2024-05-14T19:08:09.000+03:00
diff --git a/ggml-metal.m b/ggml-metal.m
@@ -1378,7 +1378,7 @@ static enum ggml_status ggml_metal_graph_compute(
                         const bool use_f16 = (src1 && src1->type == GGML_TYPE_F16);
 
                         if (ne00%4 == 0) {
-                            while (nth < ne00/4 && nth < 256) {
+                            while (nth < ne00/4 && nth*ne01*ne02*ne03 < 256) {
                                 nth *= 2;
                             }
                             if (use_f16) {
@@ -1387,7 +1387,7 @@ static enum ggml_status ggml_metal_graph_compute(
                                 pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX_F32_4].pipeline;
                             }
                         } else {
-                            while (nth < ne00 && nth < 1024) {
+                            while (nth < ne00 && nth*ne01*ne02*ne03 < 256) {
                                 nth *= 2;
                             }
                             if (use_f16) {

Original file line number	Diff line number	Diff line change
`@@ -1378,7 +1378,7 @@ static enum ggml_status ggml_metal_graph_compute(`
`1378`	`1378`	`const bool use_f16 = (src1 && src1->type == GGML_TYPE_F16);`
`1379`	`1379`
`1380`	`1380`	`if (ne00%4 == 0) {`
`1381`		`- while (nth < ne00/4 && nth < 256) {`
	`1381`	`+ while (nth < ne00/4 && nthne01ne02*ne03 < 256) {`
`1382`	`1382`	`nth *= 2;`
`1383`	`1383`	`}`
`1384`	`1384`	`if (use_f16) {`
`@@ -1387,7 +1387,7 @@ static enum ggml_status ggml_metal_graph_compute(`
`1387`	`1387`	`pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX_F32_4].pipeline;`
`1388`	`1388`	`}`
`1389`	`1389`	`} else {`
`1390`		`- while (nth < ne00 && nth < 1024) {`
	`1390`	`+ while (nth < ne00 && nthne01ne02*ne03 < 256) {`
`1391`	`1391`	`nth *= 2;`
`1392`	`1392`	`}`
`1393`	`1393`	`if (use_f16) {`