diff --git a/packages/tasks/src/model-libraries-snippets.ts b/packages/tasks/src/model-libraries-snippets.ts index 39e973b511..19c3c12b98 100644 --- a/packages/tasks/src/model-libraries-snippets.ts +++ b/packages/tasks/src/model-libraries-snippets.ts @@ -434,8 +434,63 @@ pipe = DiffusionPipeline.from_pretrained("${get_base_diffusers_model(model)}") pipe.load_textual_inversion("${model.id}")`, ]; +const diffusers_flux_fill = (model: ModelData) => [ + `import torch +from diffusers import FluxFillPipeline +from diffusers.utils import load_image + +image = load_image("https://huggingface.co/datasets/diffusers/diffusers-images-docs/resolve/main/cup.png") +mask = load_image("https://huggingface.co/datasets/diffusers/diffusers-images-docs/resolve/main/cup_mask.png") + +pipe = FluxFillPipeline.from_pretrained("${model.id}", torch_dtype=torch.bfloat16).to("cuda") +image = pipe( + prompt="a white paper cup", + image=image, + mask_image=mask, + height=1632, + width=1232, + guidance_scale=30, + num_inference_steps=50, + max_sequence_length=512, + generator=torch.Generator("cpu").manual_seed(0) +).images[0] +image.save(f"flux-fill-dev.png")`, +]; + +const diffusers_inpainting = (model: ModelData) => [ + `import torch +from diffusers import AutoPipelineForInpainting +from diffusers.utils import load_image + +pipe = AutoPipelineForInpainting.from_pretrained("${model.id}", torch_dtype=torch.float16, variant="fp16").to("cuda") + +img_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png" +mask_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png" + +image = load_image(img_url).resize((1024, 1024)) +mask_image = load_image(mask_url).resize((1024, 1024)) + +prompt = "a tiger sitting on a park bench" +generator = torch.Generator(device="cuda").manual_seed(0) + +image = pipe( + prompt=prompt, + image=image, + mask_image=mask_image, + guidance_scale=8.0, + num_inference_steps=20, # steps between 15 and 30 work well for us + strength=0.99, # make sure to use \`strength\` below 1.0 + generator=generator, +).images[0]`, +]; + export const diffusers = (model: ModelData): string[] => { - if (model.tags.includes("controlnet")) { + if ( + model.tags.includes("StableDiffusionInpaintPipeline") || + model.tags.includes("StableDiffusionXLInpaintPipeline") + ) { + return diffusers_inpainting(model); + } else if (model.tags.includes("controlnet")) { return diffusers_controlnet(model); } else if (model.tags.includes("lora")) { if (model.pipeline_tag === "image-to-image") { @@ -449,6 +504,8 @@ export const diffusers = (model: ModelData): string[] => { } } else if (model.tags.includes("textual_inversion")) { return diffusers_textual_inversion(model); + } else if (model.tags.includes("FluxFillPipeline")) { + return diffusers_flux_fill(model); } else if (model.pipeline_tag === "image-to-video") { return diffusers_image_to_video(model); } else if (model.pipeline_tag === "image-to-image") { diff --git a/packages/tasks/src/model-libraries.ts b/packages/tasks/src/model-libraries.ts index 366301aa1a..faefea9ed7 100644 --- a/packages/tasks/src/model-libraries.ts +++ b/packages/tasks/src/model-libraries.ts @@ -465,12 +465,6 @@ export const MODEL_LIBRARIES_UI_ELEMENTS = { repoUrl: "https://github.com/Tencent/HunyuanDiT", countDownloads: `path:"pytorch_model_ema.pt" OR path:"pytorch_model_distill.pt"`, }, - "hunyuan3d-2": { - prettyLabel: "Hunyuan3D-2", - repoName: "Hunyuan3D-2", - repoUrl: "https://github.com/Tencent/Hunyuan3D-2", - countDownloads: `path_filename:"model_index" OR path_filename:"config"`, - }, imstoucan: { prettyLabel: "IMS Toucan", repoName: "IMS-Toucan",