Fix layer patch dtype selection for CLIP text encoder models.

Author: RyanJDick

invokeai/app/invocations/compel.py

@@ -86,7 +86,7 @@ def _lora_loader() -> Iterator[Tuple[ModelPatchRaw, float]]:
                 model=text_encoder,
                 patches=_lora_loader(),
                 prefix="lora_te_",
-                dtype=TorchDevice.choose_torch_dtype(),
+                dtype=text_encoder.dtype,
                 cached_weights=cached_weights,
             ),
             # Apply CLIP Skip after LoRA to prevent LoRA application from failing on skipped layers.
@@ -184,7 +184,7 @@ def _lora_loader() -> Iterator[Tuple[ModelPatchRaw, float]]:
                 model=text_encoder,
                 patches=_lora_loader(),
                 prefix=lora_prefix,
-                dtype=TorchDevice.choose_torch_dtype(),
+                dtype=text_encoder.dtype,
                 cached_weights=cached_weights,
             ),
             # Apply CLIP Skip after LoRA to prevent LoRA application from failing on skipped layers.

invokeai/app/invocations/flux_text_encoder.py

@@ -22,7 +22,6 @@
 from invokeai.backend.patches.lora_conversions.flux_lora_constants import FLUX_LORA_CLIP_PREFIX
 from invokeai.backend.patches.model_patch_raw import ModelPatchRaw
 from invokeai.backend.stable_diffusion.diffusion.conditioning_data import ConditioningFieldData, FLUXConditioningInfo
-from invokeai.backend.util.devices import TorchDevice
 
 
 @invocation(
@@ -116,7 +115,7 @@ def _clip_encode(self, context: InvocationContext) -> torch.Tensor:
                         model=clip_text_encoder,
                         patches=self._clip_lora_iterator(context),
                         prefix=FLUX_LORA_CLIP_PREFIX,
-                        dtype=TorchDevice.choose_torch_dtype(),
+                        dtype=clip_text_encoder.dtype,
                         cached_weights=cached_weights,
                     )
                 )

invokeai/app/invocations/sd3_text_encoder.py

@@ -21,7 +21,6 @@
 from invokeai.backend.patches.lora_conversions.flux_lora_constants import FLUX_LORA_CLIP_PREFIX
 from invokeai.backend.patches.model_patch_raw import ModelPatchRaw
 from invokeai.backend.stable_diffusion.diffusion.conditioning_data import ConditioningFieldData, SD3ConditioningInfo
-from invokeai.backend.util.devices import TorchDevice
 
 # The SD3 T5 Max Sequence Length set based on the default in diffusers.
 SD3_T5_MAX_SEQ_LEN = 256
@@ -155,7 +154,7 @@ def _clip_encode(
                         model=clip_text_encoder,
                         patches=self._clip_lora_iterator(context, clip_model),
                         prefix=FLUX_LORA_CLIP_PREFIX,
-                        dtype=TorchDevice.choose_torch_dtype(),
+                        dtype=clip_text_encoder.dtype,
                         cached_weights=cached_weights,
                     )
                 )