load safetensor back to numpy (#1986)

Author: lvyufeng

.github/workflows/ci_pipeline.yaml

@@ -86,7 +86,7 @@ jobs:
     needs: pylint-check
     strategy:
       matrix:
-        ms_version: ['2.2.14', '2.3.1']
+        ms_version: ['2.2.14', '2.3.1', '2.4.10', '2.5.0']
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3

mindnlp/core/serialization.py

@@ -1410,18 +1410,14 @@ def convert(info: dict[str, Any]):
         assert end - begin == math.prod(shape) * np.dtype(numpy_dtype).itemsize
         buf = byte_buf[begin:end]
 
-        try:
-            if info['dtype'] == 'BF16' and not SUPPORT_BF16:
-                raise ValueError('not support bfloat16.')
-            out = Tensor.convert_bytes_to_tensor(buf, tuple(shape), ms_dtype)
-        except:
-            array = np.frombuffer(buf, dtype=numpy_dtype).reshape(shape)
-
-            if array.dtype == bfloat16 and not SUPPORT_BF16:
-                logger.warning_once("MindSpore do not support bfloat16 dtype, we will automaticlly convert to float16")
-                array = array.astype(np.float16)
-            array = array.astype(array.dtype)
-            out = Tensor.from_numpy(array)
+
+        array = np.frombuffer(buf, dtype=numpy_dtype).reshape(shape)
+
+        if array.dtype == bfloat16 and not SUPPORT_BF16:
+            logger.warning_once("MindSpore do not support bfloat16 dtype, we will automaticlly convert to float16")
+            array = array.astype(np.float16)
+        array = array.astype(array.dtype)
+        out = Tensor.from_numpy(array)
         return out
 
     with open(filename, "rb") as fp: