Skip to content

Commit 613e77f

Browse files
CyberVyCyberVy
authored
Fix Callback Tensor Inputs of the SDXL Controlnet Inpaint and Img2img Pipelines are missing "controlnet_image". (#10880)
* Update pipeline_controlnet_inpaint_sd_xl.py * Update pipeline_controlnet_sd_xl_img2img.py * Update pipeline_controlnet_union_inpaint_sd_xl.py * Update pipeline_controlnet_union_sd_xl_img2img.py * Update pipeline_controlnet_inpaint_sd_xl.py * Update pipeline_controlnet_sd_xl_img2img.py * Update pipeline_controlnet_union_inpaint_sd_xl.py * Update pipeline_controlnet_union_sd_xl_img2img.py * Apply make style and make fix-copies fixes * Update geodiff_molecule_conformation.ipynb * Delete examples/research_projects/geodiff/geodiff_molecule_conformation.ipynb * Delete examples/research_projects/gligen/demo.ipynb * Create geodiff_molecule_conformation.ipynb * Create demo.ipynb * Update geodiff_molecule_conformation.ipynb * Update geodiff_molecule_conformation.ipynb * Delete examples/research_projects/geodiff/geodiff_molecule_conformation.ipynb * Add files via upload * Delete src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py * Add files via upload
1 parent 1450c2a commit 613e77f

File tree

4 files changed

+13
-11
lines changed

4 files changed

+13
-11
lines changed

‎src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -237,6 +237,7 @@ class StableDiffusionXLControlNetInpaintPipeline(
237237
"add_neg_time_ids",
238238
"mask",
239239
"masked_image_latents",
240+
"control_image",
240241
]
241242

242243
def __init__(
@@ -743,15 +744,15 @@ def check_inputs(
743744
if padding_mask_crop is not None:
744745
if not isinstance(image, PIL.Image.Image):
745746
raise ValueError(
746-
f"The image should be a PIL image when inpainting mask crop, but is of type" f" {type(image)}."
747+
f"The image should be a PIL image when inpainting mask crop, but is of type {type(image)}."
747748
)
748749
if not isinstance(mask_image, PIL.Image.Image):
749750
raise ValueError(
750751
f"The mask image should be a PIL image when inpainting mask crop, but is of type"
751752
f" {type(mask_image)}."
752753
)
753754
if output_type != "pil":
754-
raise ValueError(f"The output type should be PIL when inpainting mask crop, but is" f" {output_type}.")
755+
raise ValueError(f"The output type should be PIL when inpainting mask crop, but is {output_type}.")
755756

756757
if prompt_embeds is not None and pooled_prompt_embeds is None:
757758
raise ValueError(
@@ -1644,7 +1645,7 @@ def denoising_value_valid(dnv):
16441645
f"Incorrect configuration settings! The config of `pipeline.unet`: {self.unet.config} expects"
16451646
f" {self.unet.config.in_channels} but received `num_channels_latents`: {num_channels_latents} +"
16461647
f" `num_channels_mask`: {num_channels_mask} + `num_channels_masked_image`: {num_channels_masked_image}"
1647-
f" = {num_channels_latents+num_channels_masked_image+num_channels_mask}. Please verify the config of"
1648+
f" = {num_channels_latents + num_channels_masked_image + num_channels_mask}. Please verify the config of"
16481649
" `pipeline.unet` or your `mask_image` or `image` input."
16491650
)
16501651
elif num_channels_unet != 4:
@@ -1835,6 +1836,7 @@ def denoising_value_valid(dnv):
18351836
latents = callback_outputs.pop("latents", latents)
18361837
prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
18371838
negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds)
1839+
control_image = callback_outputs.pop("control_image", control_image)
18381840

18391841
# call the callback, if provided
18401842
if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):

‎src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl_img2img.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -242,6 +242,7 @@ class StableDiffusionXLControlNetImg2ImgPipeline(
242242
"add_time_ids",
243243
"negative_pooled_prompt_embeds",
244244
"add_neg_time_ids",
245+
"control_image",
245246
]
246247

247248
def __init__(
@@ -1614,6 +1615,7 @@ def __call__(
16141615
)
16151616
add_time_ids = callback_outputs.pop("add_time_ids", add_time_ids)
16161617
add_neg_time_ids = callback_outputs.pop("add_neg_time_ids", add_neg_time_ids)
1618+
control_image = callback_outputs.pop("control_image", control_image)
16171619

16181620
# call the callback, if provided
16191621
if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):

‎src/diffusers/pipelines/controlnet/pipeline_controlnet_union_inpaint_sd_xl.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -219,6 +219,7 @@ class StableDiffusionXLControlNetUnionInpaintPipeline(
219219
"add_time_ids",
220220
"mask",
221221
"masked_image_latents",
222+
"control_image",
222223
]
223224

224225
def __init__(
@@ -726,15 +727,15 @@ def check_inputs(
726727
if padding_mask_crop is not None:
727728
if not isinstance(image, PIL.Image.Image):
728729
raise ValueError(
729-
f"The image should be a PIL image when inpainting mask crop, but is of type" f" {type(image)}."
730+
f"The image should be a PIL image when inpainting mask crop, but is of type {type(image)}."
730731
)
731732
if not isinstance(mask_image, PIL.Image.Image):
732733
raise ValueError(
733734
f"The mask image should be a PIL image when inpainting mask crop, but is of type"
734735
f" {type(mask_image)}."
735736
)
736737
if output_type != "pil":
737-
raise ValueError(f"The output type should be PIL when inpainting mask crop, but is" f" {output_type}.")
738+
raise ValueError(f"The output type should be PIL when inpainting mask crop, but is {output_type}.")
738739

739740
if prompt_embeds is not None and pooled_prompt_embeds is None:
740741
raise ValueError(
@@ -1743,6 +1744,7 @@ def denoising_value_valid(dnv):
17431744
latents = callback_outputs.pop("latents", latents)
17441745
prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
17451746
negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds)
1747+
control_image = callback_outputs.pop("control_image", control_image)
17461748

17471749
# call the callback, if provided
17481750
if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):

‎src/diffusers/pipelines/controlnet/pipeline_controlnet_union_sd_xl_img2img.py

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -252,12 +252,7 @@ class StableDiffusionXLControlNetUnionImg2ImgPipeline(
252252
"feature_extractor",
253253
"image_encoder",
254254
]
255-
_callback_tensor_inputs = [
256-
"latents",
257-
"prompt_embeds",
258-
"add_text_embeds",
259-
"add_time_ids",
260-
]
255+
_callback_tensor_inputs = ["latents", "prompt_embeds", "add_text_embeds", "add_time_ids", "control_image"]
261256

262257
def __init__(
263258
self,
@@ -1562,6 +1557,7 @@ def __call__(
15621557
prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
15631558
add_text_embeds = callback_outputs.pop("add_text_embeds", add_text_embeds)
15641559
add_time_ids = callback_outputs.pop("add_time_ids", add_time_ids)
1560+
control_image = callback_outputs.pop("control_image", control_image)
15651561

15661562
# call the callback, if provided
15671563
if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):

0 commit comments

Comments
 (0)