add spacemit impl

Change-Id: I62403816a869fefabdfc2b7d5655daa7ef6a8504

Author: alex-spacemit

README-SpacemiT-EN.md

@@ -0,0 +1,27 @@
+## Intruction
+This repository is a RISCV derivative developed by SpacemiT in the community version of llama.cpp, using the Mlas library in the SpacemiT version of ONNXRuntime to replace some of the performance bottlenecks in the implementation of ggml.c. LLM can be used to obtain a great performance improvement in Q4_0 quantization.
+
+## Build
+~~~ bash
+# Go to the llama.cpp directory and execute the script, you can change the number of threads, Release or Debug compilation options.
+bash scripts/build-riscv64-spacemit.sh
+~~~
+
+## Build Options Description
+This repository adds `GGML_SPACEMIT` to the GGML backend, optionally turning it on and replacing it with the SpacemiT implementation.
+
+## Quantization
+~~~ bash
+./llama-quantize --token-embedding-type q4_0  qwen2.5-3b-f32.gguf qwen2.5-3b-q4_0.gguf Q4_0
+~~~
+
+## Performance
+Performance tests were performed using llama-bench and some of the model performance data are as follows(based on SpacemiT-K1@4threads).
+
+| model name     | prefill@64t（t/s） | decode@64t(t/s) |
+| :-----:        | :----:             | :----:          |
+| qwen2.5-0.5b   | 107.10             | 16.34           |
+| qwen2.5-1.5b   | 32.11              | 5.46            |
+| tinyllama-1.1b | 40.47              | 7.55            |
+| llama3.2-1b    | 41.6               | 7.32            |
+| gemma2-2b      | 18.96              | 2.72            |

README-SpacemiT.md

@@ -0,0 +1,28 @@
+## 简介
+本仓库是进迭时空(SpacemiT)在llama.cpp社区版本上开发的RISCV衍生版本, 使用进迭时空版本ONNXRuntime中的Mlas算子库替换了ggml.c中的部分性能瓶颈实现, LLM可在Q4_0量化下获得极大性能提升
+
+## 编译构建
+~~~ bash
+# 进入llama.cpp目录执行脚本, 可自行修改线程数、Release Or Debug编译选项
+bash scripts/build-riscv64-spacemit.sh
+~~~
+
+## 编译选项说明
+本仓库在GGML编译后端的之外增加了`GGML_CPU_RISCV64_SPACEMIT`, 可选择开启并替换为SpacemiT的RVV及IME实现
+
+## 量化说明
+~~~ bash
+./llama-quantize --token-embedding-type q4_0  qwen2.5-3b-f32.gguf qwen2.5-3b-q4_0.gguf Q4_0
+# --token-embedding-type q4_0时达到最高性能，但需要考虑模型精度，建议设置为f32或f16
+~~~
+
+## 性能
+使用llama-bench进行性能测试，部分模型性能数据如下。
+
+| model name     | prefill@64t（t/s） | decode@64t(t/s) |
+| :-----:        | :----:             | :----:          |
+| qwen2.5-0.5b   | 107.10             | 16.34           |
+| qwen2.5-1.5b   | 32.11              | 5.46            |
+| tinyllama-1.1b | 40.47              | 7.55            |
+| llama3.2-1b    | 41.6               | 7.32            |
+| gemma2-2b      | 18.96              | 2.72            |

cmake/riscv64-spacemit-gcc.cmake renamed to cmake/riscv64-spacemit-linux-gnu-gcc.cmake

@@ -7,23 +7,64 @@ function(ggml_add_cpu_backend_variant_impl tag_name)
 
     ggml_add_backend_library(${GGML_CPU_NAME})
 
-    list (APPEND GGML_CPU_SOURCES
-        ggml-cpu/ggml-cpu.c
-        ggml-cpu/ggml-cpu.cpp
-        ggml-cpu/ggml-cpu-aarch64.cpp
-        ggml-cpu/ggml-cpu-aarch64.h
-        ggml-cpu/ggml-cpu-hbm.cpp
-        ggml-cpu/ggml-cpu-hbm.h
-        ggml-cpu/ggml-cpu-quants.c
-        ggml-cpu/ggml-cpu-quants.h
-        ggml-cpu/ggml-cpu-traits.cpp
-        ggml-cpu/ggml-cpu-traits.h
-        ggml-cpu/amx/amx.cpp
-        ggml-cpu/amx/amx.h
-        ggml-cpu/amx/mmq.cpp
-        ggml-cpu/amx/mmq.h
-        ggml-cpu/ggml-cpu-impl.h
+    if(GGML_CPU_RISCV64_SPACEMIT)
+        list (APPEND GGML_CPU_SOURCES
+            ggml-cpu/ggml-cpu.c
+            ggml-cpu/ggml-cpu.cpp
+            ggml-cpu/ggml-cpu-riscv64-spacemit.cpp
+            ggml-cpu/ggml-cpu-riscv64-spacemit.h
+            ggml-cpu/ggml-cpu-hbm.cpp
+            ggml-cpu/ggml-cpu-hbm.h
+            ggml-cpu/ggml-cpu-quants.c
+            ggml-cpu/ggml-cpu-quants.h
+            ggml-cpu/ggml-cpu-traits.cpp
+            ggml-cpu/ggml-cpu-traits.h
+            ggml-cpu/ggml-cpu-impl.h
         )
+        
+        if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "riscv64")
+            include(FetchContent)
+            # TODO replace with git repo
+            FetchContent_Declare(
+                onnxruntime
+                GIT_REPOSITORY ssh://$ENV{GERRIT_USER}@gerrit.dc.com:29418/DSA/onnxruntime
+                GIT_TAG "c17089e2e45067e24911d95611d2196a3dd63694"
+            )
+            # FetchContent_Declare(
+            #     onnxruntime
+            #     GIT_REPOSITORY https://github.com/space-mit/onnxruntime
+            #     GIT_TAG "d0780f050bbaaf2951d2dc0c1cc8459803068fbe"
+            # )
+
+            FetchContent_Populate(onnxruntime)
+
+            add_subdirectory(ggml-cpu/onnxruntime_mlas)
+            add_compile_definitions(BUILD_MLAS_NO_ONNXRUNTIME)
+            target_include_directories(${GGML_CPU_NAME} PRIVATE 
+                ${onnxruntime_SOURCE_DIR}/onnxruntime/core/mlas/lib
+                ${onnxruntime_SOURCE_DIR}/onnxruntime/core/mlas/inc
+            )
+            target_link_libraries(${GGML_CPU_NAME} PRIVATE onnxruntime_mlas)
+        endif()
+    else()
+        list (APPEND GGML_CPU_SOURCES
+            ggml-cpu/ggml-cpu.c
+            ggml-cpu/ggml-cpu.cpp
+            ggml-cpu/ggml-cpu-aarch64.cpp
+            ggml-cpu/ggml-cpu-aarch64.h
+            ggml-cpu/ggml-cpu-hbm.cpp
+            ggml-cpu/ggml-cpu-hbm.h
+            ggml-cpu/ggml-cpu-quants.c
+            ggml-cpu/ggml-cpu-quants.h
+            ggml-cpu/ggml-cpu-traits.cpp
+            ggml-cpu/ggml-cpu-traits.h
+            ggml-cpu/amx/amx.cpp
+            ggml-cpu/amx/amx.h
+            ggml-cpu/amx/mmq.cpp
+            ggml-cpu/amx/mmq.h
+            ggml-cpu/ggml-cpu-impl.h
+            )
+    endif()
 
     target_compile_features(${GGML_CPU_NAME} PRIVATE c_std_11 cxx_std_17)
     target_include_directories(${GGML_CPU_NAME} PRIVATE . ggml-cpu)
@@ -312,7 +353,9 @@ function(ggml_add_cpu_backend_variant_impl tag_name)
         message(STATUS "Unknown architecture")
     endif()
 
-    if (GGML_CPU_AARCH64)
+    if (GGML_CPU_RISCV64_SPACEMIT)
+        target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_USE_CPU_RISCV64_SPACEMIT)
+    elseif (GGML_CPU_AARCH64)
         target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_USE_CPU_AARCH64)
     endif()