|
| 1 | +import einops |
| 2 | +import torch |
| 3 | +from einops import repeat |
| 4 | + |
| 5 | +from invokeai.app.invocations.fields import FluxKontextConditioningField |
| 6 | +from invokeai.app.invocations.flux_vae_encode import FluxVaeEncodeInvocation |
| 7 | +from invokeai.app.invocations.model import VAEField |
| 8 | +from invokeai.app.services.shared.invocation_context import InvocationContext |
| 9 | +from invokeai.backend.flux.sampling_utils import pack |
| 10 | +from invokeai.backend.stable_diffusion.diffusers_pipeline import image_resized_to_grid_as_tensor |
| 11 | + |
| 12 | + |
| 13 | +def generate_img_ids_with_offset( |
| 14 | + h: int, w: int, batch_size: int, device: torch.device, dtype: torch.dtype, idx_offset: int = 0 |
| 15 | +) -> torch.Tensor: |
| 16 | + """Generate tensor of image position ids with an optional offset. |
| 17 | +
|
| 18 | + Args: |
| 19 | + h (int): Height of image in latent space. |
| 20 | + w (int): Width of image in latent space. |
| 21 | + batch_size (int): Batch size. |
| 22 | + device (torch.device): Device. |
| 23 | + dtype (torch.dtype): dtype. |
| 24 | + idx_offset (int): Offset to add to the first dimension of the image ids. |
| 25 | +
|
| 26 | + Returns: |
| 27 | + torch.Tensor: Image position ids. |
| 28 | + """ |
| 29 | + |
| 30 | + if device.type == "mps": |
| 31 | + orig_dtype = dtype |
| 32 | + dtype = torch.float16 |
| 33 | + |
| 34 | + img_ids = torch.zeros(h // 2, w // 2, 3, device=device, dtype=dtype) |
| 35 | + img_ids[..., 0] = idx_offset # Set the offset for the first dimension |
| 36 | + img_ids[..., 1] = img_ids[..., 1] + torch.arange(h // 2, device=device, dtype=dtype)[:, None] |
| 37 | + img_ids[..., 2] = img_ids[..., 2] + torch.arange(w // 2, device=device, dtype=dtype)[None, :] |
| 38 | + img_ids = repeat(img_ids, "h w c -> b (h w) c", b=batch_size) |
| 39 | + |
| 40 | + if device.type == "mps": |
| 41 | + img_ids = img_ids.to(orig_dtype) |
| 42 | + |
| 43 | + return img_ids |
| 44 | + |
| 45 | + |
| 46 | +class KontextExtension: |
| 47 | + """Applies FLUX Kontext (reference image) conditioning.""" |
| 48 | + |
| 49 | + def __init__( |
| 50 | + self, |
| 51 | + kontext_field: FluxKontextConditioningField, |
| 52 | + context: InvocationContext, |
| 53 | + vae_field: VAEField, |
| 54 | + device: torch.device, |
| 55 | + dtype: torch.dtype, |
| 56 | + ): |
| 57 | + """ |
| 58 | + Initializes the KontextExtension, pre-processing the reference image |
| 59 | + into latents and positional IDs. |
| 60 | + """ |
| 61 | + self._context = context |
| 62 | + self._device = device |
| 63 | + self._dtype = dtype |
| 64 | + self._vae_field = vae_field |
| 65 | + self.kontext_field = kontext_field |
| 66 | + |
| 67 | + # Pre-process and cache the kontext latents and ids upon initialization. |
| 68 | + self.kontext_latents, self.kontext_ids = self._prepare_kontext() |
| 69 | + |
| 70 | + def _prepare_kontext(self) -> tuple[torch.Tensor, torch.Tensor]: |
| 71 | + """Encodes the reference image and prepares its latents and IDs.""" |
| 72 | + image = self._context.images.get_pil(self.kontext_field.image.image_name) |
| 73 | + |
| 74 | + # Reuse VAE encoding logic from FluxVaeEncodeInvocation |
| 75 | + vae_info = self._context.models.load(self._vae_field.vae) |
| 76 | + image_tensor = image_resized_to_grid_as_tensor(image.convert("RGB")) |
| 77 | + if image_tensor.dim() == 3: |
| 78 | + image_tensor = einops.rearrange(image_tensor, "c h w -> 1 c h w") |
| 79 | + image_tensor = image_tensor.to(self._device) |
| 80 | + |
| 81 | + kontext_latents_unpacked = FluxVaeEncodeInvocation.vae_encode(vae_info=vae_info, image_tensor=image_tensor) |
| 82 | + |
| 83 | + # Pack the latents and generate IDs. The idx_offset distinguishes these |
| 84 | + # tokens from the main image's tokens, which have an index of 0. |
| 85 | + kontext_latents_packed = pack(kontext_latents_unpacked).to(self._device, self._dtype) |
| 86 | + kontext_ids = generate_img_ids_with_offset( |
| 87 | + h=kontext_latents_unpacked.shape[2], |
| 88 | + w=kontext_latents_unpacked.shape[3], |
| 89 | + batch_size=kontext_latents_unpacked.shape[0], |
| 90 | + device=self._device, |
| 91 | + dtype=self._dtype, |
| 92 | + idx_offset=1, # Distinguishes reference tokens from main image tokens |
| 93 | + ) |
| 94 | + |
| 95 | + return kontext_latents_packed, kontext_ids |
| 96 | + |
| 97 | + def apply( |
| 98 | + self, |
| 99 | + img: torch.Tensor, |
| 100 | + img_ids: torch.Tensor, |
| 101 | + ) -> tuple[torch.Tensor, torch.Tensor]: |
| 102 | + """Concatenates the pre-processed kontext data to the main image sequence.""" |
| 103 | + # Ensure batch sizes match, repeating kontext data if necessary for batch operations. |
| 104 | + if img.shape[0] != self.kontext_latents.shape[0]: |
| 105 | + self.kontext_latents = self.kontext_latents.repeat(img.shape[0], 1, 1) |
| 106 | + self.kontext_ids = self.kontext_ids.repeat(img.shape[0], 1, 1) |
| 107 | + |
| 108 | + # Concatenate along the sequence dimension (dim=1) |
| 109 | + combined_img = torch.cat([img, self.kontext_latents], dim=1) |
| 110 | + combined_img_ids = torch.cat([img_ids, self.kontext_ids], dim=1) |
| 111 | + |
| 112 | + return combined_img, combined_img_ids |
0 commit comments