pipeline.py

import torch
import math
import numpy as np
from diffusers import StableDiffusionPipeline, DPMSolverMultistepScheduler, UniPCMultistepScheduler
from diffusers.pipelines.stable_diffusion import StableDiffusionPipelineOutput
from typing import Union, Optional, List, Callable, Dict, Any, Tuple
from momentum_scheduler import (
    GHVBScheduler,
    PLMSWithHBScheduler,
    PLMSWithNTScheduler,
    MomentumDPMSolverMultistepScheduler,
    MomentumUniPCMultistepScheduler,
)

available_solvers = {
    "GHVB": GHVBScheduler,
    "PLMS_HB": PLMSWithHBScheduler,
    "PLMS_NT": PLMSWithNTScheduler,
    "DPM-Solver++": MomentumDPMSolverMultistepScheduler,
    "UniPC": MomentumUniPCMultistepScheduler,
}

def get_momentum_number(order, beta):
    out = order if beta == 1.0 else order - (1 - beta)
    return out

def setup_scheduler(pipe, scheduler, momentum_type="Polyak's heavy ball", order=4.0, beta=1.0, original_config=None):
    assert original_config is not None
    
    if scheduler in ["DPM-Solver++", "UniPC"]:
        if momentum_type in ["Nesterov"]:
            raise NotImplementedError(f"{scheduler} w/ Nesterov is not implemented.")

        pipe.scheduler = available_solvers[scheduler].from_config(original_config)
        pipe.scheduler.initialize_momentum(beta=beta)

    elif scheduler in ["PLMS"]:
        momentum_number = get_momentum_number(order, beta)
        method = "PLMS_HB" if momentum_type == "Polyak's heavy ball" else "PLMS_NT"
        pipe.scheduler = DPMSolverMultistepScheduler.from_config(original_config)
        pipe.init_scheduler(method=method, order=momentum_number)
        pipe.clear_scheduler()

    elif scheduler in ["GHVB"]:
        momentum_number = get_momentum_number(order, beta)
        pipe.scheduler = DPMSolverMultistepScheduler.from_config(original_config)
        pipe.init_scheduler(method="GHVB", order=momentum_number)
        pipe.clear_scheduler()

    return pipe

class CustomPipeline(StableDiffusionPipeline):
    def clear_scheduler(self):
        self.scheduler_uncond.clear()
        self.scheduler_text.clear()
    
    def init_scheduler(self, method, order):
        # equivalent to not applied numerical operator splitting since orders are the same
        self.scheduler_uncond = available_solvers[method](self.scheduler, order)
        self.scheduler_text = available_solvers[method](self.scheduler, order)

    def get_noise(self, latents, prompt_embeds, guidance_scale, t, do_classifier_free_guidance):
        # expand the latents if we are doing classifier free guidance
        latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents
        latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)

        # predict the noise residual
        noise_pred = self.unet(latent_model_input, t, encoder_hidden_states=prompt_embeds).sample

        if do_classifier_free_guidance:
            noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
            grads_a = guidance_scale * (noise_pred_text - noise_pred_uncond)

        return noise_pred_uncond, grads_a

    def denoising_step(
            self,
            latents,
            prompt_embeds,
            guidance_scale,
            t,
            do_classifier_free_guidance,
            method,
            extra_step_kwargs,
    ):
        noise_pred_uncond, grads_a = self.get_noise(
            latents, prompt_embeds, guidance_scale, t, do_classifier_free_guidance
        )
        if method in ["dpm", "unipc"]:
            latents = self.scheduler.step(noise_pred_uncond + grads_a, t, latents, **extra_step_kwargs).prev_sample
        
        elif method in ["hb", "ghvb", "nt"]:
            latents = self.scheduler_uncond.step(noise_pred_uncond, t, latents, output_mode="scale")
            latents = self.scheduler_text.step(grads_a, t, latents, output_mode='back')
        else:
            raise NotImplementedError

        return latents

    @torch.no_grad()
    def __call__(
        self,
        prompt: Union[str, List[str]] = None,
        height: Optional[int] = None,
        width: Optional[int] = None,
        num_inference_steps: int = 50,
        guidance_scale: float = 7.5,
        negative_prompt: Optional[Union[str, List[str]]] = None,
        num_images_per_prompt: Optional[int] = 1,
        eta: float = 0.0,
        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,
        latents: Optional[torch.FloatTensor] = None,
        prompt_embeds: Optional[torch.FloatTensor] = None,
        negative_prompt_embeds: Optional[torch.FloatTensor] = None,
        output_type: Optional[str] = "pil",
        return_dict: bool = True,
        callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,
        callback_steps: int = 1,
        cross_attention_kwargs: Optional[Dict[str, Any]] = None,
        method="ghvb",
    ):
        # 0. Default height and width to unet
        height = height or self.unet.config.sample_size * self.vae_scale_factor
        width = width or self.unet.config.sample_size * self.vae_scale_factor

        # 1. Check inputs. Raise error if not correct
        self.check_inputs(
            prompt, height, width, callback_steps, negative_prompt, prompt_embeds, negative_prompt_embeds
        )

        # 2. Define call parameters
        if prompt is not None and isinstance(prompt, str):
            batch_size = 1
        elif prompt is not None and isinstance(prompt, list):
            batch_size = len(prompt)
        else:
            batch_size = prompt_embeds.shape[0]

        device = self._execution_device
        # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
        # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`
        # corresponds to doing no classifier free guidance.
        do_classifier_free_guidance = guidance_scale > 1.0

        # 3. Encode input prompt
        prompt_embeds = self._encode_prompt(
            prompt,
            device,
            num_images_per_prompt,
            do_classifier_free_guidance,
            negative_prompt,
            prompt_embeds=prompt_embeds,
            negative_prompt_embeds=negative_prompt_embeds,
        )

        # 4. Prepare timesteps
        self.scheduler.set_timesteps(num_inference_steps, device=device)
        timesteps = self.scheduler.timesteps
        # print(timesteps)

        # 5. Prepare latent variables
        num_channels_latents = self.unet.config.in_channels
        latents = self.prepare_latents(
            batch_size * num_images_per_prompt,
            num_channels_latents,
            height,
            width,
            prompt_embeds.dtype,
            device,
            generator,
            latents,
        )

        # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
        extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)

        # 7. Denoising loop
        num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order
        with self.progress_bar(total=num_inference_steps) as progress_bar:
            for i, t in enumerate(timesteps):
                latents = self.denoising_step(
                    latents,
                    prompt_embeds,
                    guidance_scale,
                    t,
                    do_classifier_free_guidance,
                    method,
                    extra_step_kwargs,
                )

                # call the callback, if provided
                if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
                    progress_bar.update()
                    if callback is not None and i % callback_steps == 0:
                        callback(i, t, latents)

        if output_type == "latent":
            image = latents
            has_nsfw_concept = None
        elif output_type == "pil":
            # 8. Post-processing
            image = self.decode_latents(latents)

            # 9. Run safety checker
            # image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype)
            has_nsfw_concept = False

            # 10. Convert to PIL
            image = self.numpy_to_pil(image)
        else:
            # 8. Post-processing
            image = self.decode_latents(latents)

            # 9. Run safety checker
            # image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype)
            has_nsfw_concept = False

        # Offload last model to CPU
        if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None:
            self.final_offload_hook.offload()

        if not return_dict:
            return (image, has_nsfw_concept)

        return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept)

    def generate(self, params):
        params["output_type"] = "latent"
        ori_latents = self.__call__(**params)["images"]

        with torch.no_grad():
            latents = torch.clone(ori_latents)
            image = self.decode_latents(latents)
            image = self.numpy_to_pil(image)[0]

        return image, ori_latents