fix: remove impossible dtypes

Author: OlivierDehaene

README.md

@@ -128,7 +128,7 @@ Options:
           If `dtype` is not set, it defaults to float32 on accelerate, and float16 for all other architectures
 
           [env: DTYPE=]
-          [possible values: float16, float32]
+          [possible values: float16]
 
       --pooling <POOLING>
           Optionally control the pooling method. 

backends/src/dtype.rs

@@ -6,9 +6,21 @@ use clap::ValueEnum;
 #[derive(Debug, PartialEq)]
 #[cfg_attr(feature = "clap", derive(Clone, ValueEnum))]
 pub enum DType {
-    #[cfg(any(feature = "python", feature = "candle"))]
+    // Float16 is not available on accelerate
+    #[cfg(any(
+        feature = "python",
+        all(feature = "candle", not(feature = "accelerate"))
+    ))]
     Float16,
-    #[cfg(any(feature = "python", feature = "candle"))]
+    // Float32 is not available on candle cuda
+    #[cfg(any(
+        feature = "python",
+        all(
+            feature = "candle",
+            not(feature = "flash-attn"),
+            not(feature = "flash-attn-v1")
+        )
+    ))]
     Float32,
     // #[cfg(feature = "candle")]
     // Q6K,
@@ -17,9 +29,21 @@ pub enum DType {
 impl fmt::Display for DType {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
-            #[cfg(any(feature = "python", feature = "candle"))]
+            // Float16 is not available on accelerate
+            #[cfg(any(
+                feature = "python",
+                all(feature = "candle", not(feature = "accelerate"))
+            ))]
             DType::Float16 => write!(f, "float16"),
-            #[cfg(any(feature = "python", feature = "candle"))]
+            // Float32 is not available on candle cuda
+            #[cfg(any(
+                feature = "python",
+                all(
+                    feature = "candle",
+                    not(feature = "flash-attn"),
+                    not(feature = "flash-attn-v1")
+                )
+            ))]
             DType::Float32 => write!(f, "float32"),
             // #[cfg(feature = "candle")]
             // DType::Q6K => write!(f, "q6k"),

router/Cargo.toml

@@ -54,6 +54,6 @@ mkl = ["text-embeddings-backend/mkl"]
 accelerate = ["text-embeddings-backend/accelerate"]
 python = ["text-embeddings-backend/python"]
 candle = ["text-embeddings-backend/candle"]
-candle-cuda = ["text-embeddings-backend/candle", "text-embeddings-backend/flash-attn"]
-candle-cuda-turing = ["text-embeddings-backend/candle", "text-embeddings-backend/flash-attn-v1"]
+candle-cuda = ["candle", "text-embeddings-backend/flash-attn"]
+candle-cuda-turing = ["candle", "text-embeddings-backend/flash-attn-v1"]
 static-linking = ["text-embeddings-backend/static-linking"]

router/src/main.rs

@@ -232,10 +232,14 @@ async fn main() -> Result<()> {
 
     // Get dtype
     let dtype = args.dtype.unwrap_or_else(|| {
-        if cfg!(feature = "accelerate") {
-            return DType::Float32;
+        #[cfg(feature = "accelerate")]
+        {
+            DType::Float32
+        }
+        #[cfg(not(feature = "accelerate"))]
+        {
+            DType::Float16
         }
-        DType::Float16
     });
 
     // Create backend