Merge remote-tracking branch 'ggerganov/master' into fix_decoding

* ggerganov/master: (73 commits)
  ci : disable failing CUDA and Java builds
  readme : fix references to download-ggml-model.sh (ggml-org#2427)
  make : remove "talk" target until updated
  ggml : add ggml-cpu-impl.h (skip) (#0)
  sync : ggml
  talk-llama : sync llama.cpp
  ggml : add AVX512DQ requirement for AVX512 builds (llama/9622)
  log : add CONT level for continuing previous log entry (llama/9610)
  threads: fix msvc build without openmp (llama/9615)
  cuda: add q8_0->f32 cpy operation (llama/9571)
  threads: improve ggml_barrier scaling with large number of threads (llama/9598)
  ggml : AVX512 gemm for Q4_0_8_8 (llama/9532)
  metal : use F32 prec for K*Q in vec FA (llama/9595)
  Revert "[SYCL] fallback mmvq (ggml/9088)" (llama/9579)
  musa: enable building fat binaries, enable unified memory, and disable Flash Attention on QY1 (MTT S80) (llama/9526)
  Fix merge error in #9454 (llama/9589)
  CUDA: enable Gemma FA for HIP/Pascal (llama/9581)
  RWKV v6: RWKV_WKV op CUDA implementation (llama/9454)
  ggml-alloc : fix list of allocated tensors with GGML_ALLOCATOR_DEBUG (llama/9573)
  Update CUDA graph on scale change plus clear nodes/params (llama/9550)
  ...

Author: bygreencn

.devops/cublas.Dockerfile

@@ -12,7 +12,7 @@ FROM ${BASE_CUDA_DEV_CONTAINER} as build
 ARG CUDA_DOCKER_ARCH=all
 
 RUN apt-get update && \
-    apt-get install -y build-essential git cmake
+    apt-get install -y build-essential git cmake libsdl2-dev
 
 WORKDIR /app
 

.devops/main-cuda.Dockerfile

@@ -17,7 +17,7 @@ ENV CUDA_DOCKER_ARCH=${CUDA_DOCKER_ARCH}
 ENV GGML_CUDA=1
 
 RUN apt-get update && \
-    apt-get install -y build-essential \
+    apt-get install -y build-essential libsdl2-dev \
     && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
 
 # Ref: https://stackoverflow.com/a/53464012

.devops/main.Dockerfile

@@ -12,7 +12,7 @@ FROM ubuntu:22.04 AS runtime
 WORKDIR /app
 
 RUN apt-get update && \
-  apt-get install -y curl ffmpeg \
+  apt-get install -y curl ffmpeg libsdl2-dev \
   && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
 
 COPY --from=build /app /app

.github/workflows/bindings-go.yml

@@ -13,10 +13,10 @@ jobs:
   ubuntu-latest:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v5
         with:
-          go-version: '^1.19'
-      - uses: actions/checkout@v1
+          go-version: '^1.23'
+      - uses: actions/checkout@v4
       - run: |
           cd bindings/go
           make test

.github/workflows/build.yml

@@ -586,30 +586,31 @@ jobs:
           cd whisper/examples/whisper.android
           ./gradlew assembleRelease --no-daemon -PGGML_HOME=$PATH_TO_GGML
 
-  android_java:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Clone
-        uses: actions/checkout@v4
-
-      - name: set up JDK 11
-        uses: actions/setup-java@v4
-        with:
-          java-version: '11'
-          distribution: 'temurin'
-          cache: gradle
-
-      - name: Setup Android SDK
-        uses: android-actions/setup-android@v3
-        with:
-          cmdline-tools-version: 9.0
-
-      - name: Build
-        run: |
-          cd examples/whisper.android.java
-          chmod +x ./gradlew
-          ./gradlew assembleRelease
+# TODO: disable because of following fail: https://github.com/ggerganov/whisper.cpp/actions/runs/11019444420/job/30627193602
+#  android_java:
+#    runs-on: ubuntu-latest
+#
+#    steps:
+#      - name: Clone
+#        uses: actions/checkout@v4
+#
+#      - name: set up JDK 11
+#        uses: actions/setup-java@v4
+#        with:
+#          java-version: '11'
+#          distribution: 'temurin'
+#          cache: gradle
+#
+#      - name: Setup Android SDK
+#        uses: android-actions/setup-android@v3
+#        with:
+#          cmdline-tools-version: 9.0
+#
+#      - name: Build
+#        run: |
+#          cd examples/whisper.android.java
+#          chmod +x ./gradlew
+#          ./gradlew assembleRelease
 
 # TODO: disabled because of following fail: https://github.com/ggerganov/whisper.cpp/actions/runs/9686220096/job/26735899598
 #  java:

.github/workflows/docker.yml

@@ -18,7 +18,9 @@ jobs:
       matrix:
         config:
           - { tag: "main", dockerfile: ".devops/main.Dockerfile", platform: "linux/amd64,linux/arm64" }
-          - { tag: "main-cuda", dockerfile: ".devops/main-cuda.Dockerfile", platform: "linux/amd64" }
+          #TODO: the cuda image keeps failing - disable for now
+          #      https://github.com/ggerganov/whisper.cpp/actions/runs/11019444428/job/30602020339
+          #- { tag: "main-cuda", dockerfile: ".devops/main-cuda.Dockerfile", platform: "linux/amd64" }
 
     steps:
       - name: Check out the repo

Makefile

@@ -141,8 +141,8 @@ else
 		command \
 		stream \
 		lsp \
-		talk \
 		talk-llama
+	# talk (TODO: disalbed)
 endif
 
 default: $(BUILD_TARGETS)
@@ -1080,10 +1080,12 @@ lsp: examples/lsp/lsp.cpp \
 	$(CXX) $(CXXFLAGS) $(CFLAGS_SDL) -c $< -o $(call GET_OBJ_FILE, $<)
 	$(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) $(LDFLAGS_SDL)
 
-talk: examples/talk/talk.cpp examples/talk/gpt-2.cpp \
-	$(OBJ_GGML) $(OBJ_WHISPER) $(OBJ_COMMON) $(OBJ_SDL)
-	$(CXX) $(CXXFLAGS) $(CFLAGS_SDL) -c $< -o $(call GET_OBJ_FILE, $<)
-	$(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) $(LDFLAGS_SDL)
+# TODO: disabled until update
+#       https://github.com/ggerganov/whisper.cpp/issues/1818
+#talk: examples/talk/talk.cpp examples/talk/gpt-2.cpp \
+#	$(OBJ_GGML) $(OBJ_WHISPER) $(OBJ_COMMON) $(OBJ_SDL)
+#	$(CXX) $(CXXFLAGS) $(CFLAGS_SDL) -c $< -o $(call GET_OBJ_FILE, $<)
+#	$(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS) $(LDFLAGS_SDL)
 
 talk-llama: examples/talk-llama/talk-llama.cpp examples/talk-llama/llama.cpp examples/talk-llama/llama-vocab.cpp examples/talk-llama/llama-grammar.cpp examples/talk-llama/llama-sampling.cpp examples/talk-llama/unicode.cpp examples/talk-llama/unicode-data.cpp \
 	$(OBJ_GGML) $(OBJ_WHISPER) $(OBJ_COMMON) $(OBJ_SDL)

README.md

@@ -21,6 +21,7 @@ High-performance inference of [OpenAI's Whisper](https://github.com/openai/whisp
 - Support for CPU-only inference
 - [Efficient GPU support for NVIDIA](https://github.com/ggerganov/whisper.cpp#nvidia-gpu-support-via-cublas)
 - [OpenVINO Support](https://github.com/ggerganov/whisper.cpp#openvino-support)
+- [Ascend NPU Support](https://github.com/ggerganov/whisper.cpp#ascend-npu-support)
 - [C-style API](https://github.com/ggerganov/whisper.cpp/blob/master/include/whisper.h)
 
 Supported platforms:
@@ -74,7 +75,7 @@ git clone https://github.com/ggerganov/whisper.cpp.git
 Then, download one of the Whisper [models](models/README.md) converted in [`ggml` format](#ggml-format). For example:
 
 ```bash
-bash ./models/download-ggml-model.sh base.en
+sh ./models/download-ggml-model.sh base.en
 ```
 
 Now build the [main](examples/main) example and transcribe an audio file like this:
@@ -145,7 +146,7 @@ options:
   -ng,       --no-gpu            [false  ] disable GPU
 
 
-bash ./models/download-ggml-model.sh base.en
+sh ./models/download-ggml-model.sh base.en
 Downloading ggml model base.en ...
 ggml-base.en.bin               100%[========================>] 141.11M  6.34MB/s    in 24s
 Done! Model 'base.en' saved in 'models/ggml-base.en.bin'
@@ -448,6 +449,39 @@ cmake -DWHISPER_MKL=ON ..
 WHISPER_MKL=1 make -j
 ```
 
+## Ascend NPU support
+
+Ascend NPU provides inference acceleration via [`CANN`](https://www.hiascend.com/en/software/cann) and AI cores. 
+
+First, check if your Ascend NPU device is supported:
+
+**Verified devices**
+| Ascend NPU                    | Status  |
+|:-----------------------------:|:-------:|
+| Atlas 300T A2                 | Support |
+
+Then, make sure you have installed [`CANN toolkit`](https://www.hiascend.com/en/software/cann/community) . The lasted version of CANN is recommanded.
+
+Now build `whisper.cpp` with CANN support:
+
+```
+mkdir build
+cd build
+cmake .. -D GGML_CANN=on
+make -j
+```
+
+Run the inference examples as usual, for example:
+
+```
+./build/bin/main -f samples/jfk.wav -m models/ggml-base.en.bin -t 8
+```
+
+*Notes:*
+
+- If you have trouble with Ascend NPU device, please create a issue with **[CANN]** prefix/tag.
+- If you run successfully with your Ascend NPU device, please help update the table `Verified devices`.
+
 ## Docker
 
 ### Prerequisites

bindings/go/Makefile

@@ -17,6 +17,11 @@ EXAMPLES_DIR := $(wildcard examples/*)
 INCLUDE_PATH := $(abspath ../../include):$(abspath ../../ggml/include)
 LIBRARY_PATH := $(abspath ../..)
 
+ifeq ($(GGML_CUDA),1)
+	LIBRARY_PATH := $(LIBRARY_PATH):$(CUDA_PATH)/targets/$(UNAME_M)-linux/lib/
+	BUILD_FLAGS := -ldflags "-extldflags '-lcudart -lcuda -lcublas'"
+endif
+
 ifeq ($(UNAME_S),Darwin)
 	EXT_LDFLAGS := -framework Foundation -framework Metal -framework MetalKit
 endif

bindings/go/README.md

@@ -62,6 +62,12 @@ This will compile a static `libwhisper.a` in a `build` folder, download a model
 make examples
 ```
 
+To build using cuda support add `GGML_CUDA=1`:
+
+```bash
+GGML_CUDA=1 make examples
+```
+
 The examples are placed in the `build` directory. Once built, you can download all the models with the following command:
 
 ```bash