import{S as wm,i as gm,s as $m,e as a,k as p,w as d,t as i,M as Em,c as o,d as s,m as f,a as n,x as _,h as r,b as h,G as t,g as m,y as v,q as y,o as b,B as w,v as jm}from"../../chunks/vendor-hf-doc-builder.js";import{T as gn}from"../../chunks/Tip-hf-doc-builder.js";import{I as N}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as A}from"../../chunks/CodeBlock-hf-doc-builder.js";function km(x){let u;return{c(){u=i("It is strongly discouraged to make use of [`torch.autocast`](https://pytorch.org/docs/stable/amp.html#torch.autocast) in any of the pipelines as it can lead to black images and is always slower than using pure \n  float16 precision.")},l(c){u=r(c,"It is strongly discouraged to make use of [`torch.autocast`](https://pytorch.org/docs/stable/amp.html#torch.autocast) in any of the pipelines as it can lead to black images and is always slower than using pure \n  float16 precision.")},m(c,g){m(c,u,g)},d(c){c&&s(u)}}}function Tm(x){let u;return{c(){u=i(`Attention slicing is useful even if a batch size of just 1 is used - as long
  as the model uses more than one attention head. If there is more than one
  attention head the *QK^T* attention matrix can be computed sequentially for
  each head which can save a significant amount of memory.`)},l(c){u=r(c,`Attention slicing is useful even if a batch size of just 1 is used - as long
  as the model uses more than one attention head. If there is more than one
  attention head the *QK^T* attention matrix can be computed sequentially for
  each head which can save a significant amount of memory.`)},m(c,g){m(c,u,g)},d(c){c&&s(u)}}}function Pm(x){let u,c,g,P;return{c(){u=i("Consider using "),c=a("a"),g=i("model offloading"),P=i(" as another point in the optimization space: it will be much faster, but memory savings won't be as large."),this.h()},l($){u=r($,"Consider using "),c=o($,"A",{href:!0});var k=n(c);g=r(k,"model offloading"),k.forEach(s),P=r($," as another point in the optimization space: it will be much faster, but memory savings won't be as large."),this.h()},h(){h(c,"href","#model_offloading")},m($,k){m($,u,k),m($,c,k),t(c,g),m($,P,k)},d($){$&&s(u),$&&s(c),$&&s(P)}}}function Dm(x){let u;return{c(){u=i("This feature requires `accelerate` version 0.17.0 or larger.")},l(c){u=r(c,"This feature requires `accelerate` version 0.17.0 or larger.")},m(c,g){m(c,u,g)},d(c){c&&s(u)}}}function Am(x){let u,c,g,P,$,k,$n,ls,En,Ra,S,jn,ps,kn,Tn,Fe,Pn,Dn,dt,An,Sn,za,_t,qn,Ma,me,fs,R,Ha,Nn,ms,xn,Un,hs,In,Cn,j,z,cs,Fn,Gn,us,Rn,zn,ds,Mn,Hn,M,_s,Vn,On,vs,Bn,Ln,ys,Xn,Yn,H,bs,Wn,Kn,ws,Qn,Jn,gs,Zn,ei,V,$s,ti,si,Es,ai,oi,js,ni,ii,O,ks,ri,li,Ts,pi,fi,Ps,mi,hi,B,Ds,ci,ui,As,di,_i,Ss,vi,Va,vt,yi,Oa,L,he,qs,Ge,bi,Ns,wi,Ba,Re,ze,gi,$i,La,ce,Ei,xs,ji,ki,Xa,Me,Ya,X,ue,Us,He,Ti,Is,Pi,Wa,de,Di,Ve,Ai,Si,Ka,Oe,Qa,Y,_e,Cs,Be,qi,Fs,Ni,Ja,U,xi,Gs,Ui,Ii,Rs,Ci,Fi,Za,Le,eo,ve,to,W,ye,zs,Xe,Gi,Ms,Ri,so,yt,zi,ao,be,oo,we,Mi,bt,Hi,Vi,no,Ye,io,wt,Oi,ro,K,ge,Hs,We,Bi,Vs,Li,lo,gt,Xi,po,I,Yi,$t,Wi,Ki,Et,Qi,Ji,fo,$e,Zi,jt,er,tr,mo,Ke,ho,kt,sr,co,Tt,uo,Q,Ee,Os,Qe,ar,Bs,or,_o,Pt,nr,vo,je,ir,Dt,rr,lr,yo,Je,bo,At,pr,wo,ke,fr,Ls,mr,hr,go,Te,$o,St,cr,Eo,Ze,jo,D,Xs,ur,dr,Ys,_r,vr,Ws,yr,br,et,wr,gr,ko,qt,To,J,Pe,Ks,tt,$r,Qs,Er,Po,st,Nt,jr,kr,Do,C,Tr,Js,Pr,Dr,Zs,Ar,Sr,Ao,xt,qr,So,De,Nr,ea,xr,Ur,qo,at,No,Ut,Ir,xo,ot,Uo,Ae,Io,Z,Se,ta,nt,Cr,sa,Fr,Co,It,Gr,Fo,Ct,Rr,Go,it,Ro,ee,qe,aa,rt,zr,oa,Mr,zo,Ne,Hr,na,Vr,Or,Mo,Ft,Br,Ho,lt,Vo,xe,Lr,ia,Xr,Yr,Oo,pt,Bo,te,Ue,ra,ft,Wr,la,Kr,Lo,F,Qr,mt,Jr,Zr,ht,el,tl,Xo,Gt,sl,Yo,Ie,pa,se,fa,al,ol,ma,nl,il,ha,rl,ll,E,ae,ca,pl,fl,ua,ml,hl,da,cl,ul,oe,_a,dl,_l,va,vl,yl,ya,bl,wl,ne,ba,gl,$l,wa,El,jl,ga,kl,Tl,ie,$a,Pl,Dl,Ea,Al,Sl,ja,ql,Nl,re,ka,xl,Ul,Ta,Il,Cl,Pa,Fl,Gl,le,Da,Rl,zl,Aa,Ml,Hl,Sa,Vl,Ol,pe,qa,Bl,Ll,Na,Xl,Yl,xa,Wl,Wo,Rt,Kl,Ko,G,Ua,Ql,Jl,Ia,Zl,ep,zt,Mt,tp,sp,Qo,ct,Jo;return k=new N({}),Ge=new N({}),Me=new A({props:{code:`import torch

torch.backends.cudnn.benchmark = True`,highlighted:`<span class="hljs-keyword">import</span> torch

torch.backends.cudnn.benchmark = <span class="hljs-literal">True</span>`}}),He=new N({}),Oe=new A({props:{code:`import torch

torch.backends.cuda.matmul.allow_tf32 = True`,highlighted:`<span class="hljs-keyword">import</span> torch

torch.backends.cuda.matmul.allow_tf32 = <span class="hljs-literal">True</span>`}}),Be=new N({}),Le=new A({props:{code:`pipe = StableDiffusionPipeline.from_pretrained(
    "runwayml/stable-diffusion-v1-5",
    
    torch_dtype=torch.float16,
)
pipe = pipe.to("cuda")

prompt = "a photo of an astronaut riding a horse on mars"
image = pipe(prompt).images[0]`,highlighted:`pipe = StableDiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;runwayml/stable-diffusion-v1-5&quot;</span>,
    
    torch_dtype=torch.float16,
)
pipe = pipe.to(<span class="hljs-string">&quot;cuda&quot;</span>)

prompt = <span class="hljs-string">&quot;a photo of an astronaut riding a horse on mars&quot;</span>
image = pipe(prompt).images[<span class="hljs-number">0</span>]`}}),ve=new gn({props:{warning:!0,$$slots:{default:[km]},$$scope:{ctx:x}}}),Xe=new N({}),be=new gn({props:{$$slots:{default:[Tm]},$$scope:{ctx:x}}}),Ye=new A({props:{code:`import torch
from diffusers import StableDiffusionPipeline

pipe = StableDiffusionPipeline.from_pretrained(
    "runwayml/stable-diffusion-v1-5",
    
    torch_dtype=torch.float16,
)
pipe = pipe.to("cuda")

prompt = "a photo of an astronaut riding a horse on mars"
pipe.enable_attention_slicing()
image = pipe(prompt).images[0]`,highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline

pipe = StableDiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;runwayml/stable-diffusion-v1-5&quot;</span>,
    
    torch_dtype=torch.float16,
)
pipe = pipe.to(<span class="hljs-string">&quot;cuda&quot;</span>)

prompt = <span class="hljs-string">&quot;a photo of an astronaut riding a horse on mars&quot;</span>
pipe.enable_attention_slicing()
image = pipe(prompt).images[<span class="hljs-number">0</span>]`}}),We=new N({}),Ke=new A({props:{code:`import torch
from diffusers import StableDiffusionPipeline

pipe = StableDiffusionPipeline.from_pretrained(
    "runwayml/stable-diffusion-v1-5",
    
    torch_dtype=torch.float16,
)
pipe = pipe.to("cuda")

prompt = "a photo of an astronaut riding a horse on mars"
pipe.enable_vae_slicing()
images = pipe([prompt] * 32).images`,highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline

pipe = StableDiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;runwayml/stable-diffusion-v1-5&quot;</span>,
    
    torch_dtype=torch.float16,
)
pipe = pipe.to(<span class="hljs-string">&quot;cuda&quot;</span>)

prompt = <span class="hljs-string">&quot;a photo of an astronaut riding a horse on mars&quot;</span>
pipe.enable_vae_slicing()
images = pipe([prompt] * <span class="hljs-number">32</span>).images`}}),Qe=new N({}),Je=new A({props:{code:`import torch
from diffusers import StableDiffusionPipeline

pipe = StableDiffusionPipeline.from_pretrained(
    "runwayml/stable-diffusion-v1-5",
    
    torch_dtype=torch.float16,
)

prompt = "a photo of an astronaut riding a horse on mars"
pipe.enable_sequential_cpu_offload()
image = pipe(prompt).images[0]`,highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline

pipe = StableDiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;runwayml/stable-diffusion-v1-5&quot;</span>,
    
    torch_dtype=torch.float16,
)

prompt = <span class="hljs-string">&quot;a photo of an astronaut riding a horse on mars&quot;</span>
pipe.enable_sequential_cpu_offload()
image = pipe(prompt).images[<span class="hljs-number">0</span>]`}}),Te=new gn({props:{$$slots:{default:[Pm]},$$scope:{ctx:x}}}),Ze=new A({props:{code:`import torch
from diffusers import StableDiffusionPipeline

pipe = StableDiffusionPipeline.from_pretrained(
    "runwayml/stable-diffusion-v1-5",
    
    torch_dtype=torch.float16,
)

prompt = "a photo of an astronaut riding a horse on mars"
pipe.enable_sequential_cpu_offload()
pipe.enable_attention_slicing(1)

image = pipe(prompt).images[0]`,highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline

pipe = StableDiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;runwayml/stable-diffusion-v1-5&quot;</span>,
    
    torch_dtype=torch.float16,
)

prompt = <span class="hljs-string">&quot;a photo of an astronaut riding a horse on mars&quot;</span>
pipe.enable_sequential_cpu_offload()
pipe.enable_attention_slicing(<span class="hljs-number">1</span>)

image = pipe(prompt).images[<span class="hljs-number">0</span>]`}}),tt=new N({}),at=new A({props:{code:`import torch
from diffusers import StableDiffusionPipeline

pipe = StableDiffusionPipeline.from_pretrained(
    "runwayml/stable-diffusion-v1-5",  
    torch_dtype=torch.float16,
)

prompt = "a photo of an astronaut riding a horse on mars"
pipe.enable_model_cpu_offload()
image = pipe(prompt).images[0]`,highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline

pipe = StableDiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;runwayml/stable-diffusion-v1-5&quot;</span>,  
    torch_dtype=torch.float16,
)

prompt = <span class="hljs-string">&quot;a photo of an astronaut riding a horse on mars&quot;</span>
pipe.enable_model_cpu_offload()
image = pipe(prompt).images[<span class="hljs-number">0</span>]`}}),ot=new A({props:{code:`import torch
from diffusers import StableDiffusionPipeline

pipe = StableDiffusionPipeline.from_pretrained(
    "runwayml/stable-diffusion-v1-5",
    torch_dtype=torch.float16,
)

prompt = "a photo of an astronaut riding a horse on mars"
pipe.enable_model_cpu_offload()
pipe.enable_attention_slicing(1)

image = pipe(prompt).images[0]`,highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline

pipe = StableDiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;runwayml/stable-diffusion-v1-5&quot;</span>,
    torch_dtype=torch.float16,
)

prompt = <span class="hljs-string">&quot;a photo of an astronaut riding a horse on mars&quot;</span>
pipe.enable_model_cpu_offload()
pipe.enable_attention_slicing(<span class="hljs-number">1</span>)

image = pipe(prompt).images[<span class="hljs-number">0</span>]`}}),Ae=new gn({props:{$$slots:{default:[Dm]},$$scope:{ctx:x}}}),nt=new N({}),it=new A({props:{code:`print(pipe.unet.conv_out.state_dict()["weight"].stride())  # (2880, 9, 3, 1)
pipe.unet.to(memory_format=torch.channels_last)  # in-place operation
print(
    pipe.unet.conv_out.state_dict()["weight"].stride()
)  # (2880, 1, 960, 320) having a stride of 1 for the 2nd dimension proves that it works`,highlighted:`<span class="hljs-built_in">print</span>(pipe.unet.conv_out.state_dict()[<span class="hljs-string">&quot;weight&quot;</span>].stride())  <span class="hljs-comment"># (2880, 9, 3, 1)</span>
pipe.unet.to(memory_format=torch.channels_last)  <span class="hljs-comment"># in-place operation</span>
<span class="hljs-built_in">print</span>(
    pipe.unet.conv_out.state_dict()[<span class="hljs-string">&quot;weight&quot;</span>].stride()
)  <span class="hljs-comment"># (2880, 1, 960, 320) having a stride of 1 for the 2nd dimension proves that it works</span>`}}),rt=new N({}),lt=new A({props:{code:`import time
import torch
from diffusers import StableDiffusionPipeline
import functools

# torch disable grad
torch.set_grad_enabled(False)

# set variables
n_experiments = 2
unet_runs_per_experiment = 50


# load inputs
def generate_inputs():
    sample = torch.randn(2, 4, 64, 64).half().cuda()
    timestep = torch.rand(1).half().cuda() * 999
    encoder_hidden_states = torch.randn(2, 77, 768).half().cuda()
    return sample, timestep, encoder_hidden_states


pipe = StableDiffusionPipeline.from_pretrained(
    "runwayml/stable-diffusion-v1-5",
    torch_dtype=torch.float16,
).to("cuda")
unet = pipe.unet
unet.eval()
unet.to(memory_format=torch.channels_last)  # use channels_last memory format
unet.forward = functools.partial(unet.forward, return_dict=False)  # set return_dict=False as default

# warmup
for _ in range(3):
    with torch.inference_mode():
        inputs = generate_inputs()
        orig_output = unet(*inputs)

# trace
print("tracing..")
unet_traced = torch.jit.trace(unet, inputs)
unet_traced.eval()
print("done tracing")


# warmup and optimize graph
for _ in range(5):
    with torch.inference_mode():
        inputs = generate_inputs()
        orig_output = unet_traced(*inputs)


# benchmarking
with torch.inference_mode():
    for _ in range(n_experiments):
        torch.cuda.synchronize()
        start_time = time.time()
        for _ in range(unet_runs_per_experiment):
            orig_output = unet_traced(*inputs)
        torch.cuda.synchronize()
        print(f"unet traced inference took {time.time() - start_time:.2f} seconds")
    for _ in range(n_experiments):
        torch.cuda.synchronize()
        start_time = time.time()
        for _ in range(unet_runs_per_experiment):
            orig_output = unet(*inputs)
        torch.cuda.synchronize()
        print(f"unet inference took {time.time() - start_time:.2f} seconds")

# save the model
unet_traced.save("unet_traced.pt")`,highlighted:`<span class="hljs-keyword">import</span> time
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline
<span class="hljs-keyword">import</span> functools

<span class="hljs-comment"># torch disable grad</span>
torch.set_grad_enabled(<span class="hljs-literal">False</span>)

<span class="hljs-comment"># set variables</span>
n_experiments = <span class="hljs-number">2</span>
unet_runs_per_experiment = <span class="hljs-number">50</span>


<span class="hljs-comment"># load inputs</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">generate_inputs</span>():
    sample = torch.randn(<span class="hljs-number">2</span>, <span class="hljs-number">4</span>, <span class="hljs-number">64</span>, <span class="hljs-number">64</span>).half().cuda()
    timestep = torch.rand(<span class="hljs-number">1</span>).half().cuda() * <span class="hljs-number">999</span>
    encoder_hidden_states = torch.randn(<span class="hljs-number">2</span>, <span class="hljs-number">77</span>, <span class="hljs-number">768</span>).half().cuda()
    <span class="hljs-keyword">return</span> sample, timestep, encoder_hidden_states


pipe = StableDiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;runwayml/stable-diffusion-v1-5&quot;</span>,
    torch_dtype=torch.float16,
).to(<span class="hljs-string">&quot;cuda&quot;</span>)
unet = pipe.unet
unet.<span class="hljs-built_in">eval</span>()
unet.to(memory_format=torch.channels_last)  <span class="hljs-comment"># use channels_last memory format</span>
unet.forward = functools.partial(unet.forward, return_dict=<span class="hljs-literal">False</span>)  <span class="hljs-comment"># set return_dict=False as default</span>

<span class="hljs-comment"># warmup</span>
<span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">3</span>):
    <span class="hljs-keyword">with</span> torch.inference_mode():
        inputs = generate_inputs()
        orig_output = unet(*inputs)

<span class="hljs-comment"># trace</span>
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;tracing..&quot;</span>)
unet_traced = torch.jit.trace(unet, inputs)
unet_traced.<span class="hljs-built_in">eval</span>()
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;done tracing&quot;</span>)


<span class="hljs-comment"># warmup and optimize graph</span>
<span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">5</span>):
    <span class="hljs-keyword">with</span> torch.inference_mode():
        inputs = generate_inputs()
        orig_output = unet_traced(*inputs)


<span class="hljs-comment"># benchmarking</span>
<span class="hljs-keyword">with</span> torch.inference_mode():
    <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(n_experiments):
        torch.cuda.synchronize()
        start_time = time.time()
        <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(unet_runs_per_experiment):
            orig_output = unet_traced(*inputs)
        torch.cuda.synchronize()
        <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;unet traced inference took <span class="hljs-subst">{time.time() - start_time:<span class="hljs-number">.2</span>f}</span> seconds&quot;</span>)
    <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(n_experiments):
        torch.cuda.synchronize()
        start_time = time.time()
        <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(unet_runs_per_experiment):
            orig_output = unet(*inputs)
        torch.cuda.synchronize()
        <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;unet inference took <span class="hljs-subst">{time.time() - start_time:<span class="hljs-number">.2</span>f}</span> seconds&quot;</span>)

<span class="hljs-comment"># save the model</span>
unet_traced.save(<span class="hljs-string">&quot;unet_traced.pt&quot;</span>)`}}),pt=new A({props:{code:`from diffusers import StableDiffusionPipeline
import torch
from dataclasses import dataclass


@dataclass
class UNet2DConditionOutput:
    sample: torch.FloatTensor


pipe = StableDiffusionPipeline.from_pretrained(
    "runwayml/stable-diffusion-v1-5",
    torch_dtype=torch.float16,
).to("cuda")

# use jitted unet
unet_traced = torch.jit.load("unet_traced.pt")


# del pipe.unet
class TracedUNet(torch.nn.Module):
    def __init__(self):
        super().__init__()
        self.in_channels = pipe.unet.in_channels
        self.device = pipe.unet.device

    def forward(self, latent_model_input, t, encoder_hidden_states):
        sample = unet_traced(latent_model_input, t, encoder_hidden_states)[0]
        return UNet2DConditionOutput(sample=sample)


pipe.unet = TracedUNet()

with torch.inference_mode():
    image = pipe([prompt] * 1, num_inference_steps=50).images[0]`,highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> dataclasses <span class="hljs-keyword">import</span> dataclass


<span class="hljs-meta">@dataclass</span>
<span class="hljs-keyword">class</span> <span class="hljs-title class_">UNet2DConditionOutput</span>:
    sample: torch.FloatTensor


pipe = StableDiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;runwayml/stable-diffusion-v1-5&quot;</span>,
    torch_dtype=torch.float16,
).to(<span class="hljs-string">&quot;cuda&quot;</span>)

<span class="hljs-comment"># use jitted unet</span>
unet_traced = torch.jit.load(<span class="hljs-string">&quot;unet_traced.pt&quot;</span>)


<span class="hljs-comment"># del pipe.unet</span>
<span class="hljs-keyword">class</span> <span class="hljs-title class_">TracedUNet</span>(torch.nn.Module):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self</span>):
        <span class="hljs-built_in">super</span>().__init__()
        self.in_channels = pipe.unet.in_channels
        self.device = pipe.unet.device

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, latent_model_input, t, encoder_hidden_states</span>):
        sample = unet_traced(latent_model_input, t, encoder_hidden_states)[<span class="hljs-number">0</span>]
        <span class="hljs-keyword">return</span> UNet2DConditionOutput(sample=sample)


pipe.unet = TracedUNet()

<span class="hljs-keyword">with</span> torch.inference_mode():
    image = pipe([prompt] * <span class="hljs-number">1</span>, num_inference_steps=<span class="hljs-number">50</span>).images[<span class="hljs-number">0</span>]`}}),ft=new N({}),ct=new A({props:{code:`from diffusers import StableDiffusionPipeline
import torch

pipe = StableDiffusionPipeline.from_pretrained(
    "runwayml/stable-diffusion-v1-5",
    torch_dtype=torch.float16,
).to("cuda")

pipe.enable_xformers_memory_efficient_attention()

with torch.inference_mode():
    sample = pipe("a small cat")

# optional: You can disable it via
# pipe.disable_xformers_memory_efficient_attention()`,highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline
<span class="hljs-keyword">import</span> torch

pipe = StableDiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;runwayml/stable-diffusion-v1-5&quot;</span>,
    torch_dtype=torch.float16,
).to(<span class="hljs-string">&quot;cuda&quot;</span>)

pipe.enable_xformers_memory_efficient_attention()

<span class="hljs-keyword">with</span> torch.inference_mode():
    sample = pipe(<span class="hljs-string">&quot;a small cat&quot;</span>)

<span class="hljs-comment"># optional: You can disable it via</span>
<span class="hljs-comment"># pipe.disable_xformers_memory_efficient_attention()</span>`}}),{c(){u=a("meta"),c=p(),g=a("h1"),P=a("a"),$=a("span"),d(k.$$.fragment),$n=p(),ls=a("span"),En=i("Memory and speed"),Ra=p(),S=a("p"),jn=i("We present some techniques and ideas to optimize \u{1F917} Diffusers "),ps=a("em"),kn=i("inference"),Tn=i(" for memory or speed. As a general rule, we recommend the use of "),Fe=a("a"),Pn=i("xFormers"),Dn=i(" for memory efficient attention, please see the recommended "),dt=a("a"),An=i("installation instructions"),Sn=i("."),za=p(),_t=a("p"),qn=i("We\u2019ll discuss how the following settings impact performance and memory."),Ma=p(),me=a("table"),fs=a("thead"),R=a("tr"),Ha=a("th"),Nn=p(),ms=a("th"),xn=i("Latency"),Un=p(),hs=a("th"),In=i("Speedup"),Cn=p(),j=a("tbody"),z=a("tr"),cs=a("td"),Fn=i("original"),Gn=p(),us=a("td"),Rn=i("9.50s"),zn=p(),ds=a("td"),Mn=i("x1"),Hn=p(),M=a("tr"),_s=a("td"),Vn=i("cuDNN auto-tuner"),On=p(),vs=a("td"),Bn=i("9.37s"),Ln=p(),ys=a("td"),Xn=i("x1.01"),Yn=p(),H=a("tr"),bs=a("td"),Wn=i("fp16"),Kn=p(),ws=a("td"),Qn=i("3.61s"),Jn=p(),gs=a("td"),Zn=i("x2.63"),ei=p(),V=a("tr"),$s=a("td"),ti=i("channels last"),si=p(),Es=a("td"),ai=i("3.30s"),oi=p(),js=a("td"),ni=i("x2.88"),ii=p(),O=a("tr"),ks=a("td"),ri=i("traced UNet"),li=p(),Ts=a("td"),pi=i("3.21s"),fi=p(),Ps=a("td"),mi=i("x2.96"),hi=p(),B=a("tr"),Ds=a("td"),ci=i("memory efficient attention"),ui=p(),As=a("td"),di=i("2.63s"),_i=p(),Ss=a("td"),vi=i("x3.61"),Va=p(),vt=a("em"),yi=i(`obtained on NVIDIA TITAN RTX by generating a single image of size 512x512 from
  the prompt "a photo of an astronaut riding a horse on mars" with 50 DDIM
  steps.`),Oa=p(),L=a("h2"),he=a("a"),qs=a("span"),d(Ge.$$.fragment),bi=p(),Ns=a("span"),wi=i("Enable cuDNN auto-tuner"),Ba=p(),Re=a("p"),ze=a("a"),gi=i("NVIDIA cuDNN"),$i=i("\xA0supports many algorithms to compute a convolution. Autotuner runs a short benchmark and selects the kernel with the best performance on a given hardware for a given input size."),La=p(),ce=a("p"),Ei=i("Since we\u2019re using "),xs=a("strong"),ji=i("convolutional networks"),ki=i(" (other types currently not supported), we can enable cuDNN autotuner before launching the inference by setting:"),Xa=p(),d(Me.$$.fragment),Ya=p(),X=a("h3"),ue=a("a"),Us=a("span"),d(He.$$.fragment),Ti=p(),Is=a("span"),Pi=i("Use tf32 instead of fp32 (on Ampere and later CUDA devices)"),Wa=p(),de=a("p"),Di=i("On Ampere and later CUDA devices matrix multiplications and convolutions can use the TensorFloat32 (TF32) mode for faster but slightly less accurate computations. By default PyTorch enables TF32 mode for convolutions but not matrix multiplications, and unless a network requires full float32 precision we recommend enabling this setting for matrix multiplications, too. It can significantly speed up computations with typically negligible loss of numerical accuracy. You can read more about it "),Ve=a("a"),Ai=i("here"),Si=i(". All you need to do is to add this before your inference:"),Ka=p(),d(Oe.$$.fragment),Qa=p(),Y=a("h2"),_e=a("a"),Cs=a("span"),d(Be.$$.fragment),qi=p(),Fs=a("span"),Ni=i("Half precision weights"),Ja=p(),U=a("p"),xi=i("To save more GPU memory and get more speed, you can load and run the model weights directly in half precision. This involves loading the float16 version of the weights, which was saved to a branch named "),Gs=a("code"),Ui=i("fp16"),Ii=i(", and telling PyTorch to use the "),Rs=a("code"),Ci=i("float16"),Fi=i(" type when loading them:"),Za=p(),d(Le.$$.fragment),eo=p(),d(ve.$$.fragment),to=p(),W=a("h2"),ye=a("a"),zs=a("span"),d(Xe.$$.fragment),Gi=p(),Ms=a("span"),Ri=i("Sliced attention for additional memory savings"),so=p(),yt=a("p"),zi=i("For even additional memory savings, you can use a sliced version of attention that performs the computation in steps instead of all at once."),ao=p(),d(be.$$.fragment),oo=p(),we=a("p"),Mi=i("To perform the attention computation sequentially over each head, you only need to invoke "),bt=a("a"),Hi=i("enable_attention_slicing()"),Vi=i(" in your pipeline before inference, like here:"),no=p(),d(Ye.$$.fragment),io=p(),wt=a("p"),Oi=i("There\u2019s a small performance penalty of about 10% slower inference times, but this method allows you to use Stable Diffusion in as little as 3.2 GB of VRAM!"),ro=p(),K=a("h2"),ge=a("a"),Hs=a("span"),d(We.$$.fragment),Bi=p(),Vs=a("span"),Li=i("Sliced VAE decode for larger batches"),lo=p(),gt=a("p"),Xi=i("To decode large batches of images with limited VRAM, or to enable batches with 32 images or more, you can use sliced VAE decode that decodes the batch latents one image at a time."),po=p(),I=a("p"),Yi=i("You likely want to couple this with "),$t=a("a"),Wi=i("enable_attention_slicing()"),Ki=i(" or "),Et=a("a"),Qi=i("enable_xformers_memory_efficient_attention()"),Ji=i(" to further minimize memory use."),fo=p(),$e=a("p"),Zi=i("To perform the VAE decode one image at a time, invoke "),jt=a("a"),er=i("enable_vae_slicing()"),tr=i(" in your pipeline before inference. For example:"),mo=p(),d(Ke.$$.fragment),ho=p(),kt=a("p"),sr=i("You may see a small performance boost in VAE decode on multi-image batches. There should be no performance impact on single-image batches."),co=p(),Tt=a("a"),uo=p(),Q=a("h2"),Ee=a("a"),Os=a("span"),d(Qe.$$.fragment),ar=p(),Bs=a("span"),or=i("Offloading to CPU with accelerate for memory savings"),_o=p(),Pt=a("p"),nr=i("For additional memory savings, you can offload the weights to CPU and only load them to GPU when performing the forward pass."),vo=p(),je=a("p"),ir=i("To perform CPU offloading, all you have to do is invoke "),Dt=a("a"),rr=i("enable_sequential_cpu_offload()"),lr=i(":"),yo=p(),d(Je.$$.fragment),bo=p(),At=a("p"),pr=i("And you can get the memory consumption to < 3GB."),wo=p(),ke=a("p"),fr=i("Note that this method works at the submodule level, not on whole models. This is the best way to minimize memory consumption, but inference is much slower due to the iterative nature of the process. The UNet component of the pipeline runs several times (as many as "),Ls=a("code"),mr=i("num_inference_steps"),hr=i("); each time, the different submodules of the UNet are sequentially onloaded and then offloaded as they are needed, so the number of memory transfers is large."),go=p(),d(Te.$$.fragment),$o=p(),St=a("p"),cr=i("It is also possible to chain offloading with attention slicing for minimal memory consumption (< 2GB)."),Eo=p(),d(Ze.$$.fragment),jo=p(),D=a("p"),Xs=a("strong"),ur=i("Note"),dr=i(": When using "),Ys=a("code"),_r=i("enable_sequential_cpu_offload()"),vr=i(", it is important to "),Ws=a("strong"),yr=i("not"),br=i(" move the pipeline to CUDA beforehand or else the gain in memory consumption will only be minimal. See "),et=a("a"),wr=i("this issue"),gr=i(" for more information."),ko=p(),qt=a("a"),To=p(),J=a("h2"),Pe=a("a"),Ks=a("span"),d(tt.$$.fragment),$r=p(),Qs=a("span"),Er=i("Model offloading for fast inference and memory savings"),Po=p(),st=a("p"),Nt=a("a"),jr=i("Sequential CPU offloading"),kr=i(", as discussed in the previous section, preserves a lot of memory but makes inference slower, because submodules are moved to GPU as needed, and immediately returned to CPU when a new module runs."),Do=p(),C=a("p"),Tr=i("Full-model offloading is an alternative that moves whole models to the GPU, instead of handling each model\u2019s constituent "),Js=a("em"),Pr=i("modules"),Dr=i(". This results in a negligible impact on inference time (compared with moving the pipeline to "),Zs=a("code"),Ar=i("cuda"),Sr=i("), while still providing some memory savings."),Ao=p(),xt=a("p"),qr=i(`In this scenario, only one of the main components of the pipeline (typically: text encoder, unet and vae)
will be in the GPU while the others wait in the CPU. Compoments like the UNet that run for multiple iterations will stay on GPU until they are no longer needed.`),So=p(),De=a("p"),Nr=i("This feature can be enabled by invoking "),ea=a("code"),xr=i("enable_model_cpu_offload()"),Ur=i(" on the pipeline, as shown below."),qo=p(),d(at.$$.fragment),No=p(),Ut=a("p"),Ir=i("This is also compatible with attention slicing for additional memory savings."),xo=p(),d(ot.$$.fragment),Uo=p(),d(Ae.$$.fragment),Io=p(),Z=a("h2"),Se=a("a"),ta=a("span"),d(nt.$$.fragment),Cr=p(),sa=a("span"),Fr=i("Using Channels Last memory format"),Co=p(),It=a("p"),Gr=i("Channels last memory format is an alternative way of ordering NCHW tensors in memory preserving dimensions ordering. Channels last tensors ordered in such a way that channels become the densest dimension (aka storing images pixel-per-pixel). Since not all operators currently support channels last format it may result in a worst performance, so it\u2019s better to try it and see if it works for your model."),Fo=p(),Ct=a("p"),Rr=i("For example, in order to set the UNet model in our pipeline to use channels last format, we can use the following:"),Go=p(),d(it.$$.fragment),Ro=p(),ee=a("h2"),qe=a("a"),aa=a("span"),d(rt.$$.fragment),zr=p(),oa=a("span"),Mr=i("Tracing"),zo=p(),Ne=a("p"),Hr=i("Tracing runs an example input tensor through your model, and captures the operations that are invoked as that input makes its way through the model\u2019s layers so that an executable or "),na=a("code"),Vr=i("ScriptFunction"),Or=i(" is returned that will be optimized using just-in-time compilation."),Mo=p(),Ft=a("p"),Br=i("To trace our UNet model, we can use the following:"),Ho=p(),d(lt.$$.fragment),Vo=p(),xe=a("p"),Lr=i("Then we can replace the "),ia=a("code"),Xr=i("unet"),Yr=i(" attribute of the pipeline with the traced model like the following"),Oo=p(),d(pt.$$.fragment),Bo=p(),te=a("h2"),Ue=a("a"),ra=a("span"),d(ft.$$.fragment),Wr=p(),la=a("span"),Kr=i("Memory Efficient Attention"),Lo=p(),F=a("p"),Qr=i("Recent work on optimizing the bandwitdh in the attention block has generated huge speed ups and gains in GPU memory usage. The most recent being Flash Attention from @tridao: "),mt=a("a"),Jr=i("code"),Zr=i(", "),ht=a("a"),el=i("paper"),tl=i("."),Xo=p(),Gt=a("p"),sl=i("Here are the speedups we obtain on a few Nvidia GPUs when running the inference at 512x512 with a batch size of 1 (one prompt):"),Yo=p(),Ie=a("table"),pa=a("thead"),se=a("tr"),fa=a("th"),al=i("GPU"),ol=p(),ma=a("th"),nl=i("Base Attention FP16"),il=p(),ha=a("th"),rl=i("Memory Efficient Attention FP16"),ll=p(),E=a("tbody"),ae=a("tr"),ca=a("td"),pl=i("NVIDIA Tesla T4"),fl=p(),ua=a("td"),ml=i("3.5it/s"),hl=p(),da=a("td"),cl=i("5.5it/s"),ul=p(),oe=a("tr"),_a=a("td"),dl=i("NVIDIA 3060 RTX"),_l=p(),va=a("td"),vl=i("4.6it/s"),yl=p(),ya=a("td"),bl=i("7.8it/s"),wl=p(),ne=a("tr"),ba=a("td"),gl=i("NVIDIA A10G"),$l=p(),wa=a("td"),El=i("8.88it/s"),jl=p(),ga=a("td"),kl=i("15.6it/s"),Tl=p(),ie=a("tr"),$a=a("td"),Pl=i("NVIDIA RTX A6000"),Dl=p(),Ea=a("td"),Al=i("11.7it/s"),Sl=p(),ja=a("td"),ql=i("21.09it/s"),Nl=p(),re=a("tr"),ka=a("td"),xl=i("NVIDIA TITAN RTX"),Ul=p(),Ta=a("td"),Il=i("12.51it/s"),Cl=p(),Pa=a("td"),Fl=i("18.22it/s"),Gl=p(),le=a("tr"),Da=a("td"),Rl=i("A100-SXM4-40GB"),zl=p(),Aa=a("td"),Ml=i("18.6it/s"),Hl=p(),Sa=a("td"),Vl=i("29.it/s"),Ol=p(),pe=a("tr"),qa=a("td"),Bl=i("A100-SXM-80GB"),Ll=p(),Na=a("td"),Xl=i("18.7it/s"),Yl=p(),xa=a("td"),Wl=i("29.5it/s"),Wo=p(),Rt=a("p"),Kl=i("To leverage it just make sure you have:"),Ko=p(),G=a("ul"),Ua=a("li"),Ql=i("PyTorch > 1.12"),Jl=p(),Ia=a("li"),Zl=i("Cuda available"),ep=p(),zt=a("li"),Mt=a("a"),tp=i("Installed the xformers library"),sp=i("."),Qo=p(),d(ct.$$.fragment),this.h()},l(e){const l=Em('[data-svelte="svelte-1phssyn"]',document.head);u=o(l,"META",{name:!0,content:!0}),l.forEach(s),c=f(e),g=o(e,"H1",{class:!0});var ut=n(g);P=o(ut,"A",{id:!0,class:!0,href:!0});var Ca=n(P);$=o(Ca,"SPAN",{});var Fa=n($);_(k.$$.fragment,Fa),Fa.forEach(s),Ca.forEach(s),$n=f(ut),ls=o(ut,"SPAN",{});var Ga=n(ls);En=r(Ga,"Memory and speed"),Ga.forEach(s),ut.forEach(s),Ra=f(e),S=o(e,"P",{});var Ce=n(S);jn=r(Ce,"We present some techniques and ideas to optimize \u{1F917} Diffusers "),ps=o(Ce,"EM",{});var ip=n(ps);kn=r(ip,"inference"),ip.forEach(s),Tn=r(Ce," for memory or speed. As a general rule, we recommend the use of "),Fe=o(Ce,"A",{href:!0,rel:!0});var rp=n(Fe);Pn=r(rp,"xFormers"),rp.forEach(s),Dn=r(Ce," for memory efficient attention, please see the recommended "),dt=o(Ce,"A",{href:!0});var lp=n(dt);An=r(lp,"installation instructions"),lp.forEach(s),Sn=r(Ce,"."),Ce.forEach(s),za=f(e),_t=o(e,"P",{});var pp=n(_t);qn=r(pp,"We\u2019ll discuss how the following settings impact performance and memory."),pp.forEach(s),Ma=f(e),me=o(e,"TABLE",{});var Zo=n(me);fs=o(Zo,"THEAD",{});var fp=n(fs);R=o(fp,"TR",{});var Ht=n(R);Ha=o(Ht,"TH",{}),n(Ha).forEach(s),Nn=f(Ht),ms=o(Ht,"TH",{});var mp=n(ms);xn=r(mp,"Latency"),mp.forEach(s),Un=f(Ht),hs=o(Ht,"TH",{});var hp=n(hs);In=r(hp,"Speedup"),hp.forEach(s),Ht.forEach(s),fp.forEach(s),Cn=f(Zo),j=o(Zo,"TBODY",{});var q=n(j);z=o(q,"TR",{});var Vt=n(z);cs=o(Vt,"TD",{});var cp=n(cs);Fn=r(cp,"original"),cp.forEach(s),Gn=f(Vt),us=o(Vt,"TD",{});var up=n(us);Rn=r(up,"9.50s"),up.forEach(s),zn=f(Vt),ds=o(Vt,"TD",{});var dp=n(ds);Mn=r(dp,"x1"),dp.forEach(s),Vt.forEach(s),Hn=f(q),M=o(q,"TR",{});var Ot=n(M);_s=o(Ot,"TD",{});var _p=n(_s);Vn=r(_p,"cuDNN auto-tuner"),_p.forEach(s),On=f(Ot),vs=o(Ot,"TD",{});var vp=n(vs);Bn=r(vp,"9.37s"),vp.forEach(s),Ln=f(Ot),ys=o(Ot,"TD",{});var yp=n(ys);Xn=r(yp,"x1.01"),yp.forEach(s),Ot.forEach(s),Yn=f(q),H=o(q,"TR",{});var Bt=n(H);bs=o(Bt,"TD",{});var bp=n(bs);Wn=r(bp,"fp16"),bp.forEach(s),Kn=f(Bt),ws=o(Bt,"TD",{});var wp=n(ws);Qn=r(wp,"3.61s"),wp.forEach(s),Jn=f(Bt),gs=o(Bt,"TD",{});var gp=n(gs);Zn=r(gp,"x2.63"),gp.forEach(s),Bt.forEach(s),ei=f(q),V=o(q,"TR",{});var Lt=n(V);$s=o(Lt,"TD",{});var $p=n($s);ti=r($p,"channels last"),$p.forEach(s),si=f(Lt),Es=o(Lt,"TD",{});var Ep=n(Es);ai=r(Ep,"3.30s"),Ep.forEach(s),oi=f(Lt),js=o(Lt,"TD",{});var jp=n(js);ni=r(jp,"x2.88"),jp.forEach(s),Lt.forEach(s),ii=f(q),O=o(q,"TR",{});var Xt=n(O);ks=o(Xt,"TD",{});var kp=n(ks);ri=r(kp,"traced UNet"),kp.forEach(s),li=f(Xt),Ts=o(Xt,"TD",{});var Tp=n(Ts);pi=r(Tp,"3.21s"),Tp.forEach(s),fi=f(Xt),Ps=o(Xt,"TD",{});var Pp=n(Ps);mi=r(Pp,"x2.96"),Pp.forEach(s),Xt.forEach(s),hi=f(q),B=o(q,"TR",{});var Yt=n(B);Ds=o(Yt,"TD",{});var Dp=n(Ds);ci=r(Dp,"memory efficient attention"),Dp.forEach(s),ui=f(Yt),As=o(Yt,"TD",{});var Ap=n(As);di=r(Ap,"2.63s"),Ap.forEach(s),_i=f(Yt),Ss=o(Yt,"TD",{});var Sp=n(Ss);vi=r(Sp,"x3.61"),Sp.forEach(s),Yt.forEach(s),q.forEach(s),Zo.forEach(s),Va=f(e),vt=o(e,"EM",{});var qp=n(vt);yi=r(qp,`obtained on NVIDIA TITAN RTX by generating a single image of size 512x512 from
  the prompt "a photo of an astronaut riding a horse on mars" with 50 DDIM
  steps.`),qp.forEach(s),Oa=f(e),L=o(e,"H2",{class:!0});var en=n(L);he=o(en,"A",{id:!0,class:!0,href:!0});var Np=n(he);qs=o(Np,"SPAN",{});var xp=n(qs);_(Ge.$$.fragment,xp),xp.forEach(s),Np.forEach(s),bi=f(en),Ns=o(en,"SPAN",{});var Up=n(Ns);wi=r(Up,"Enable cuDNN auto-tuner"),Up.forEach(s),en.forEach(s),Ba=f(e),Re=o(e,"P",{});var ap=n(Re);ze=o(ap,"A",{href:!0,rel:!0});var Ip=n(ze);gi=r(Ip,"NVIDIA cuDNN"),Ip.forEach(s),$i=r(ap,"\xA0supports many algorithms to compute a convolution. Autotuner runs a short benchmark and selects the kernel with the best performance on a given hardware for a given input size."),ap.forEach(s),La=f(e),ce=o(e,"P",{});var tn=n(ce);Ei=r(tn,"Since we\u2019re using "),xs=o(tn,"STRONG",{});var Cp=n(xs);ji=r(Cp,"convolutional networks"),Cp.forEach(s),ki=r(tn," (other types currently not supported), we can enable cuDNN autotuner before launching the inference by setting:"),tn.forEach(s),Xa=f(e),_(Me.$$.fragment,e),Ya=f(e),X=o(e,"H3",{class:!0});var sn=n(X);ue=o(sn,"A",{id:!0,class:!0,href:!0});var Fp=n(ue);Us=o(Fp,"SPAN",{});var Gp=n(Us);_(He.$$.fragment,Gp),Gp.forEach(s),Fp.forEach(s),Ti=f(sn),Is=o(sn,"SPAN",{});var Rp=n(Is);Pi=r(Rp,"Use tf32 instead of fp32 (on Ampere and later CUDA devices)"),Rp.forEach(s),sn.forEach(s),Wa=f(e),de=o(e,"P",{});var an=n(de);Di=r(an,"On Ampere and later CUDA devices matrix multiplications and convolutions can use the TensorFloat32 (TF32) mode for faster but slightly less accurate computations. By default PyTorch enables TF32 mode for convolutions but not matrix multiplications, and unless a network requires full float32 precision we recommend enabling this setting for matrix multiplications, too. It can significantly speed up computations with typically negligible loss of numerical accuracy. You can read more about it "),Ve=o(an,"A",{href:!0,rel:!0});var zp=n(Ve);Ai=r(zp,"here"),zp.forEach(s),Si=r(an,". All you need to do is to add this before your inference:"),an.forEach(s),Ka=f(e),_(Oe.$$.fragment,e),Qa=f(e),Y=o(e,"H2",{class:!0});var on=n(Y);_e=o(on,"A",{id:!0,class:!0,href:!0});var Mp=n(_e);Cs=o(Mp,"SPAN",{});var Hp=n(Cs);_(Be.$$.fragment,Hp),Hp.forEach(s),Mp.forEach(s),qi=f(on),Fs=o(on,"SPAN",{});var Vp=n(Fs);Ni=r(Vp,"Half precision weights"),Vp.forEach(s),on.forEach(s),Ja=f(e),U=o(e,"P",{});var Wt=n(U);xi=r(Wt,"To save more GPU memory and get more speed, you can load and run the model weights directly in half precision. This involves loading the float16 version of the weights, which was saved to a branch named "),Gs=o(Wt,"CODE",{});var Op=n(Gs);Ui=r(Op,"fp16"),Op.forEach(s),Ii=r(Wt,", and telling PyTorch to use the "),Rs=o(Wt,"CODE",{});var Bp=n(Rs);Ci=r(Bp,"float16"),Bp.forEach(s),Fi=r(Wt," type when loading them:"),Wt.forEach(s),Za=f(e),_(Le.$$.fragment,e),eo=f(e),_(ve.$$.fragment,e),to=f(e),W=o(e,"H2",{class:!0});var nn=n(W);ye=o(nn,"A",{id:!0,class:!0,href:!0});var Lp=n(ye);zs=o(Lp,"SPAN",{});var Xp=n(zs);_(Xe.$$.fragment,Xp),Xp.forEach(s),Lp.forEach(s),Gi=f(nn),Ms=o(nn,"SPAN",{});var Yp=n(Ms);Ri=r(Yp,"Sliced attention for additional memory savings"),Yp.forEach(s),nn.forEach(s),so=f(e),yt=o(e,"P",{});var Wp=n(yt);zi=r(Wp,"For even additional memory savings, you can use a sliced version of attention that performs the computation in steps instead of all at once."),Wp.forEach(s),ao=f(e),_(be.$$.fragment,e),oo=f(e),we=o(e,"P",{});var rn=n(we);Mi=r(rn,"To perform the attention computation sequentially over each head, you only need to invoke "),bt=o(rn,"A",{href:!0});var Kp=n(bt);Hi=r(Kp,"enable_attention_slicing()"),Kp.forEach(s),Vi=r(rn," in your pipeline before inference, like here:"),rn.forEach(s),no=f(e),_(Ye.$$.fragment,e),io=f(e),wt=o(e,"P",{});var Qp=n(wt);Oi=r(Qp,"There\u2019s a small performance penalty of about 10% slower inference times, but this method allows you to use Stable Diffusion in as little as 3.2 GB of VRAM!"),Qp.forEach(s),ro=f(e),K=o(e,"H2",{class:!0});var ln=n(K);ge=o(ln,"A",{id:!0,class:!0,href:!0});var Jp=n(ge);Hs=o(Jp,"SPAN",{});var Zp=n(Hs);_(We.$$.fragment,Zp),Zp.forEach(s),Jp.forEach(s),Bi=f(ln),Vs=o(ln,"SPAN",{});var ef=n(Vs);Li=r(ef,"Sliced VAE decode for larger batches"),ef.forEach(s),ln.forEach(s),lo=f(e),gt=o(e,"P",{});var tf=n(gt);Xi=r(tf,"To decode large batches of images with limited VRAM, or to enable batches with 32 images or more, you can use sliced VAE decode that decodes the batch latents one image at a time."),tf.forEach(s),po=f(e),I=o(e,"P",{});var Kt=n(I);Yi=r(Kt,"You likely want to couple this with "),$t=o(Kt,"A",{href:!0});var sf=n($t);Wi=r(sf,"enable_attention_slicing()"),sf.forEach(s),Ki=r(Kt," or "),Et=o(Kt,"A",{href:!0});var af=n(Et);Qi=r(af,"enable_xformers_memory_efficient_attention()"),af.forEach(s),Ji=r(Kt," to further minimize memory use."),Kt.forEach(s),fo=f(e),$e=o(e,"P",{});var pn=n($e);Zi=r(pn,"To perform the VAE decode one image at a time, invoke "),jt=o(pn,"A",{href:!0});var of=n(jt);er=r(of,"enable_vae_slicing()"),of.forEach(s),tr=r(pn," in your pipeline before inference. For example:"),pn.forEach(s),mo=f(e),_(Ke.$$.fragment,e),ho=f(e),kt=o(e,"P",{});var nf=n(kt);sr=r(nf,"You may see a small performance boost in VAE decode on multi-image batches. There should be no performance impact on single-image batches."),nf.forEach(s),co=f(e),Tt=o(e,"A",{name:!0}),n(Tt).forEach(s),uo=f(e),Q=o(e,"H2",{class:!0});var fn=n(Q);Ee=o(fn,"A",{id:!0,class:!0,href:!0});var rf=n(Ee);Os=o(rf,"SPAN",{});var lf=n(Os);_(Qe.$$.fragment,lf),lf.forEach(s),rf.forEach(s),ar=f(fn),Bs=o(fn,"SPAN",{});var pf=n(Bs);or=r(pf,"Offloading to CPU with accelerate for memory savings"),pf.forEach(s),fn.forEach(s),_o=f(e),Pt=o(e,"P",{});var ff=n(Pt);nr=r(ff,"For additional memory savings, you can offload the weights to CPU and only load them to GPU when performing the forward pass."),ff.forEach(s),vo=f(e),je=o(e,"P",{});var mn=n(je);ir=r(mn,"To perform CPU offloading, all you have to do is invoke "),Dt=o(mn,"A",{href:!0});var mf=n(Dt);rr=r(mf,"enable_sequential_cpu_offload()"),mf.forEach(s),lr=r(mn,":"),mn.forEach(s),yo=f(e),_(Je.$$.fragment,e),bo=f(e),At=o(e,"P",{});var hf=n(At);pr=r(hf,"And you can get the memory consumption to < 3GB."),hf.forEach(s),wo=f(e),ke=o(e,"P",{});var hn=n(ke);fr=r(hn,"Note that this method works at the submodule level, not on whole models. This is the best way to minimize memory consumption, but inference is much slower due to the iterative nature of the process. The UNet component of the pipeline runs several times (as many as "),Ls=o(hn,"CODE",{});var cf=n(Ls);mr=r(cf,"num_inference_steps"),cf.forEach(s),hr=r(hn,"); each time, the different submodules of the UNet are sequentially onloaded and then offloaded as they are needed, so the number of memory transfers is large."),hn.forEach(s),go=f(e),_(Te.$$.fragment,e),$o=f(e),St=o(e,"P",{});var uf=n(St);cr=r(uf,"It is also possible to chain offloading with attention slicing for minimal memory consumption (< 2GB)."),uf.forEach(s),Eo=f(e),_(Ze.$$.fragment,e),jo=f(e),D=o(e,"P",{});var fe=n(D);Xs=o(fe,"STRONG",{});var df=n(Xs);ur=r(df,"Note"),df.forEach(s),dr=r(fe,": When using "),Ys=o(fe,"CODE",{});var _f=n(Ys);_r=r(_f,"enable_sequential_cpu_offload()"),_f.forEach(s),vr=r(fe,", it is important to "),Ws=o(fe,"STRONG",{});var vf=n(Ws);yr=r(vf,"not"),vf.forEach(s),br=r(fe," move the pipeline to CUDA beforehand or else the gain in memory consumption will only be minimal. See "),et=o(fe,"A",{href:!0,rel:!0});var yf=n(et);wr=r(yf,"this issue"),yf.forEach(s),gr=r(fe," for more information."),fe.forEach(s),ko=f(e),qt=o(e,"A",{name:!0}),n(qt).forEach(s),To=f(e),J=o(e,"H2",{class:!0});var cn=n(J);Pe=o(cn,"A",{id:!0,class:!0,href:!0});var bf=n(Pe);Ks=o(bf,"SPAN",{});var wf=n(Ks);_(tt.$$.fragment,wf),wf.forEach(s),bf.forEach(s),$r=f(cn),Qs=o(cn,"SPAN",{});var gf=n(Qs);Er=r(gf,"Model offloading for fast inference and memory savings"),gf.forEach(s),cn.forEach(s),Po=f(e),st=o(e,"P",{});var op=n(st);Nt=o(op,"A",{href:!0});var $f=n(Nt);jr=r($f,"Sequential CPU offloading"),$f.forEach(s),kr=r(op,", as discussed in the previous section, preserves a lot of memory but makes inference slower, because submodules are moved to GPU as needed, and immediately returned to CPU when a new module runs."),op.forEach(s),Do=f(e),C=o(e,"P",{});var Qt=n(C);Tr=r(Qt,"Full-model offloading is an alternative that moves whole models to the GPU, instead of handling each model\u2019s constituent "),Js=o(Qt,"EM",{});var Ef=n(Js);Pr=r(Ef,"modules"),Ef.forEach(s),Dr=r(Qt,". This results in a negligible impact on inference time (compared with moving the pipeline to "),Zs=o(Qt,"CODE",{});var jf=n(Zs);Ar=r(jf,"cuda"),jf.forEach(s),Sr=r(Qt,"), while still providing some memory savings."),Qt.forEach(s),Ao=f(e),xt=o(e,"P",{});var kf=n(xt);qr=r(kf,`In this scenario, only one of the main components of the pipeline (typically: text encoder, unet and vae)
will be in the GPU while the others wait in the CPU. Compoments like the UNet that run for multiple iterations will stay on GPU until they are no longer needed.`),kf.forEach(s),So=f(e),De=o(e,"P",{});var un=n(De);Nr=r(un,"This feature can be enabled by invoking "),ea=o(un,"CODE",{});var Tf=n(ea);xr=r(Tf,"enable_model_cpu_offload()"),Tf.forEach(s),Ur=r(un," on the pipeline, as shown below."),un.forEach(s),qo=f(e),_(at.$$.fragment,e),No=f(e),Ut=o(e,"P",{});var Pf=n(Ut);Ir=r(Pf,"This is also compatible with attention slicing for additional memory savings."),Pf.forEach(s),xo=f(e),_(ot.$$.fragment,e),Uo=f(e),_(Ae.$$.fragment,e),Io=f(e),Z=o(e,"H2",{class:!0});var dn=n(Z);Se=o(dn,"A",{id:!0,class:!0,href:!0});var Df=n(Se);ta=o(Df,"SPAN",{});var Af=n(ta);_(nt.$$.fragment,Af),Af.forEach(s),Df.forEach(s),Cr=f(dn),sa=o(dn,"SPAN",{});var Sf=n(sa);Fr=r(Sf,"Using Channels Last memory format"),Sf.forEach(s),dn.forEach(s),Co=f(e),It=o(e,"P",{});var qf=n(It);Gr=r(qf,"Channels last memory format is an alternative way of ordering NCHW tensors in memory preserving dimensions ordering. Channels last tensors ordered in such a way that channels become the densest dimension (aka storing images pixel-per-pixel). Since not all operators currently support channels last format it may result in a worst performance, so it\u2019s better to try it and see if it works for your model."),qf.forEach(s),Fo=f(e),Ct=o(e,"P",{});var Nf=n(Ct);Rr=r(Nf,"For example, in order to set the UNet model in our pipeline to use channels last format, we can use the following:"),Nf.forEach(s),Go=f(e),_(it.$$.fragment,e),Ro=f(e),ee=o(e,"H2",{class:!0});var _n=n(ee);qe=o(_n,"A",{id:!0,class:!0,href:!0});var xf=n(qe);aa=o(xf,"SPAN",{});var Uf=n(aa);_(rt.$$.fragment,Uf),Uf.forEach(s),xf.forEach(s),zr=f(_n),oa=o(_n,"SPAN",{});var If=n(oa);Mr=r(If,"Tracing"),If.forEach(s),_n.forEach(s),zo=f(e),Ne=o(e,"P",{});var vn=n(Ne);Hr=r(vn,"Tracing runs an example input tensor through your model, and captures the operations that are invoked as that input makes its way through the model\u2019s layers so that an executable or "),na=o(vn,"CODE",{});var Cf=n(na);Vr=r(Cf,"ScriptFunction"),Cf.forEach(s),Or=r(vn," is returned that will be optimized using just-in-time compilation."),vn.forEach(s),Mo=f(e),Ft=o(e,"P",{});var Ff=n(Ft);Br=r(Ff,"To trace our UNet model, we can use the following:"),Ff.forEach(s),Ho=f(e),_(lt.$$.fragment,e),Vo=f(e),xe=o(e,"P",{});var yn=n(xe);Lr=r(yn,"Then we can replace the "),ia=o(yn,"CODE",{});var Gf=n(ia);Xr=r(Gf,"unet"),Gf.forEach(s),Yr=r(yn," attribute of the pipeline with the traced model like the following"),yn.forEach(s),Oo=f(e),_(pt.$$.fragment,e),Bo=f(e),te=o(e,"H2",{class:!0});var bn=n(te);Ue=o(bn,"A",{id:!0,class:!0,href:!0});var Rf=n(Ue);ra=o(Rf,"SPAN",{});var zf=n(ra);_(ft.$$.fragment,zf),zf.forEach(s),Rf.forEach(s),Wr=f(bn),la=o(bn,"SPAN",{});var Mf=n(la);Kr=r(Mf,"Memory Efficient Attention"),Mf.forEach(s),bn.forEach(s),Lo=f(e),F=o(e,"P",{});var Jt=n(F);Qr=r(Jt,"Recent work on optimizing the bandwitdh in the attention block has generated huge speed ups and gains in GPU memory usage. The most recent being Flash Attention from @tridao: "),mt=o(Jt,"A",{href:!0,rel:!0});var Hf=n(mt);Jr=r(Hf,"code"),Hf.forEach(s),Zr=r(Jt,", "),ht=o(Jt,"A",{href:!0,rel:!0});var Vf=n(ht);el=r(Vf,"paper"),Vf.forEach(s),tl=r(Jt,"."),Jt.forEach(s),Xo=f(e),Gt=o(e,"P",{});var Of=n(Gt);sl=r(Of,"Here are the speedups we obtain on a few Nvidia GPUs when running the inference at 512x512 with a batch size of 1 (one prompt):"),Of.forEach(s),Yo=f(e),Ie=o(e,"TABLE",{});var wn=n(Ie);pa=o(wn,"THEAD",{});var Bf=n(pa);se=o(Bf,"TR",{});var Zt=n(se);fa=o(Zt,"TH",{});var Lf=n(fa);al=r(Lf,"GPU"),Lf.forEach(s),ol=f(Zt),ma=o(Zt,"TH",{});var Xf=n(ma);nl=r(Xf,"Base Attention FP16"),Xf.forEach(s),il=f(Zt),ha=o(Zt,"TH",{});var Yf=n(ha);rl=r(Yf,"Memory Efficient Attention FP16"),Yf.forEach(s),Zt.forEach(s),Bf.forEach(s),ll=f(wn),E=o(wn,"TBODY",{});var T=n(E);ae=o(T,"TR",{});var es=n(ae);ca=o(es,"TD",{});var Wf=n(ca);pl=r(Wf,"NVIDIA Tesla T4"),Wf.forEach(s),fl=f(es),ua=o(es,"TD",{});var Kf=n(ua);ml=r(Kf,"3.5it/s"),Kf.forEach(s),hl=f(es),da=o(es,"TD",{});var Qf=n(da);cl=r(Qf,"5.5it/s"),Qf.forEach(s),es.forEach(s),ul=f(T),oe=o(T,"TR",{});var ts=n(oe);_a=o(ts,"TD",{});var Jf=n(_a);dl=r(Jf,"NVIDIA 3060 RTX"),Jf.forEach(s),_l=f(ts),va=o(ts,"TD",{});var Zf=n(va);vl=r(Zf,"4.6it/s"),Zf.forEach(s),yl=f(ts),ya=o(ts,"TD",{});var em=n(ya);bl=r(em,"7.8it/s"),em.forEach(s),ts.forEach(s),wl=f(T),ne=o(T,"TR",{});var ss=n(ne);ba=o(ss,"TD",{});var tm=n(ba);gl=r(tm,"NVIDIA A10G"),tm.forEach(s),$l=f(ss),wa=o(ss,"TD",{});var sm=n(wa);El=r(sm,"8.88it/s"),sm.forEach(s),jl=f(ss),ga=o(ss,"TD",{});var am=n(ga);kl=r(am,"15.6it/s"),am.forEach(s),ss.forEach(s),Tl=f(T),ie=o(T,"TR",{});var as=n(ie);$a=o(as,"TD",{});var om=n($a);Pl=r(om,"NVIDIA RTX A6000"),om.forEach(s),Dl=f(as),Ea=o(as,"TD",{});var nm=n(Ea);Al=r(nm,"11.7it/s"),nm.forEach(s),Sl=f(as),ja=o(as,"TD",{});var im=n(ja);ql=r(im,"21.09it/s"),im.forEach(s),as.forEach(s),Nl=f(T),re=o(T,"TR",{});var os=n(re);ka=o(os,"TD",{});var rm=n(ka);xl=r(rm,"NVIDIA TITAN RTX"),rm.forEach(s),Ul=f(os),Ta=o(os,"TD",{});var lm=n(Ta);Il=r(lm,"12.51it/s"),lm.forEach(s),Cl=f(os),Pa=o(os,"TD",{});var pm=n(Pa);Fl=r(pm,"18.22it/s"),pm.forEach(s),os.forEach(s),Gl=f(T),le=o(T,"TR",{});var ns=n(le);Da=o(ns,"TD",{});var fm=n(Da);Rl=r(fm,"A100-SXM4-40GB"),fm.forEach(s),zl=f(ns),Aa=o(ns,"TD",{});var mm=n(Aa);Ml=r(mm,"18.6it/s"),mm.forEach(s),Hl=f(ns),Sa=o(ns,"TD",{});var hm=n(Sa);Vl=r(hm,"29.it/s"),hm.forEach(s),ns.forEach(s),Ol=f(T),pe=o(T,"TR",{});var is=n(pe);qa=o(is,"TD",{});var cm=n(qa);Bl=r(cm,"A100-SXM-80GB"),cm.forEach(s),Ll=f(is),Na=o(is,"TD",{});var um=n(Na);Xl=r(um,"18.7it/s"),um.forEach(s),Yl=f(is),xa=o(is,"TD",{});var dm=n(xa);Wl=r(dm,"29.5it/s"),dm.forEach(s),is.forEach(s),T.forEach(s),wn.forEach(s),Wo=f(e),Rt=o(e,"P",{});var _m=n(Rt);Kl=r(_m,"To leverage it just make sure you have:"),_m.forEach(s),Ko=f(e),G=o(e,"UL",{});var rs=n(G);Ua=o(rs,"LI",{});var vm=n(Ua);Ql=r(vm,"PyTorch > 1.12"),vm.forEach(s),Jl=f(rs),Ia=o(rs,"LI",{});var ym=n(Ia);Zl=r(ym,"Cuda available"),ym.forEach(s),ep=f(rs),zt=o(rs,"LI",{});var np=n(zt);Mt=o(np,"A",{href:!0});var bm=n(Mt);tp=r(bm,"Installed the xformers library"),bm.forEach(s),sp=r(np,"."),np.forEach(s),rs.forEach(s),Qo=f(e),_(ct.$$.fragment,e),this.h()},h(){h(u,"name","hf:doc:metadata"),h(u,"content",JSON.stringify(Sm)),h(P,"id","memory-and-speed"),h(P,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(P,"href","#memory-and-speed"),h(g,"class","relative group"),h(Fe,"href","https://github.com/facebookresearch/xformers"),h(Fe,"rel","nofollow"),h(dt,"href","xformers"),h(he,"id","enable-cudnn-autotuner"),h(he,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(he,"href","#enable-cudnn-autotuner"),h(L,"class","relative group"),h(ze,"href","https://developer.nvidia.com/cudnn"),h(ze,"rel","nofollow"),h(ue,"id","use-tf32-instead-of-fp32-on-ampere-and-later-cuda-devices"),h(ue,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(ue,"href","#use-tf32-instead-of-fp32-on-ampere-and-later-cuda-devices"),h(X,"class","relative group"),h(Ve,"href","https://huggingface.co/docs/transformers/v4.18.0/en/performance#tf32"),h(Ve,"rel","nofollow"),h(_e,"id","half-precision-weights"),h(_e,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(_e,"href","#half-precision-weights"),h(Y,"class","relative group"),h(ye,"id","sliced-attention-for-additional-memory-savings"),h(ye,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(ye,"href","#sliced-attention-for-additional-memory-savings"),h(W,"class","relative group"),h(bt,"href","/docs/diffusers/main/en/api/pipelines/stable_diffusion/img2img#diffusers.StableDiffusionImg2ImgPipeline.enable_attention_slicing"),h(ge,"id","sliced-vae-decode-for-larger-batches"),h(ge,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(ge,"href","#sliced-vae-decode-for-larger-batches"),h(K,"class","relative group"),h($t,"href","/docs/diffusers/main/en/api/pipelines/stable_diffusion/img2img#diffusers.StableDiffusionImg2ImgPipeline.enable_attention_slicing"),h(Et,"href","/docs/diffusers/main/en/api/pipelines/stable_diffusion/img2img#diffusers.StableDiffusionImg2ImgPipeline.enable_xformers_memory_efficient_attention"),h(jt,"href","/docs/diffusers/main/en/api/pipelines/stable_diffusion/text2img#diffusers.StableDiffusionPipeline.enable_vae_slicing"),h(Tt,"name","sequential_offloading"),h(Ee,"id","offloading-to-cpu-with-accelerate-for-memory-savings"),h(Ee,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Ee,"href","#offloading-to-cpu-with-accelerate-for-memory-savings"),h(Q,"class","relative group"),h(Dt,"href","/docs/diffusers/main/en/api/pipelines/stable_diffusion/text2img#diffusers.StableDiffusionPipeline.enable_sequential_cpu_offload"),h(et,"href","https://github.com/huggingface/diffusers/issues/1934"),h(et,"rel","nofollow"),h(qt,"name","model_offloading"),h(Pe,"id","model-offloading-for-fast-inference-and-memory-savings"),h(Pe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Pe,"href","#model-offloading-for-fast-inference-and-memory-savings"),h(J,"class","relative group"),h(Nt,"href","#sequential_offloading"),h(Se,"id","using-channels-last-memory-format"),h(Se,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Se,"href","#using-channels-last-memory-format"),h(Z,"class","relative group"),h(qe,"id","tracing"),h(qe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(qe,"href","#tracing"),h(ee,"class","relative group"),h(Ue,"id","memory-efficient-attention"),h(Ue,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Ue,"href","#memory-efficient-attention"),h(te,"class","relative group"),h(mt,"href","https://github.com/HazyResearch/flash-attention"),h(mt,"rel","nofollow"),h(ht,"href","https://arxiv.org/pdf/2205.14135.pdf"),h(ht,"rel","nofollow"),h(Mt,"href","xformers")},m(e,l){t(document.head,u),m(e,c,l),m(e,g,l),t(g,P),t(P,$),v(k,$,null),t(g,$n),t(g,ls),t(ls,En),m(e,Ra,l),m(e,S,l),t(S,jn),t(S,ps),t(ps,kn),t(S,Tn),t(S,Fe),t(Fe,Pn),t(S,Dn),t(S,dt),t(dt,An),t(S,Sn),m(e,za,l),m(e,_t,l),t(_t,qn),m(e,Ma,l),m(e,me,l),t(me,fs),t(fs,R),t(R,Ha),t(R,Nn),t(R,ms),t(ms,xn),t(R,Un),t(R,hs),t(hs,In),t(me,Cn),t(me,j),t(j,z),t(z,cs),t(cs,Fn),t(z,Gn),t(z,us),t(us,Rn),t(z,zn),t(z,ds),t(ds,Mn),t(j,Hn),t(j,M),t(M,_s),t(_s,Vn),t(M,On),t(M,vs),t(vs,Bn),t(M,Ln),t(M,ys),t(ys,Xn),t(j,Yn),t(j,H),t(H,bs),t(bs,Wn),t(H,Kn),t(H,ws),t(ws,Qn),t(H,Jn),t(H,gs),t(gs,Zn),t(j,ei),t(j,V),t(V,$s),t($s,ti),t(V,si),t(V,Es),t(Es,ai),t(V,oi),t(V,js),t(js,ni),t(j,ii),t(j,O),t(O,ks),t(ks,ri),t(O,li),t(O,Ts),t(Ts,pi),t(O,fi),t(O,Ps),t(Ps,mi),t(j,hi),t(j,B),t(B,Ds),t(Ds,ci),t(B,ui),t(B,As),t(As,di),t(B,_i),t(B,Ss),t(Ss,vi),m(e,Va,l),m(e,vt,l),t(vt,yi),m(e,Oa,l),m(e,L,l),t(L,he),t(he,qs),v(Ge,qs,null),t(L,bi),t(L,Ns),t(Ns,wi),m(e,Ba,l),m(e,Re,l),t(Re,ze),t(ze,gi),t(Re,$i),m(e,La,l),m(e,ce,l),t(ce,Ei),t(ce,xs),t(xs,ji),t(ce,ki),m(e,Xa,l),v(Me,e,l),m(e,Ya,l),m(e,X,l),t(X,ue),t(ue,Us),v(He,Us,null),t(X,Ti),t(X,Is),t(Is,Pi),m(e,Wa,l),m(e,de,l),t(de,Di),t(de,Ve),t(Ve,Ai),t(de,Si),m(e,Ka,l),v(Oe,e,l),m(e,Qa,l),m(e,Y,l),t(Y,_e),t(_e,Cs),v(Be,Cs,null),t(Y,qi),t(Y,Fs),t(Fs,Ni),m(e,Ja,l),m(e,U,l),t(U,xi),t(U,Gs),t(Gs,Ui),t(U,Ii),t(U,Rs),t(Rs,Ci),t(U,Fi),m(e,Za,l),v(Le,e,l),m(e,eo,l),v(ve,e,l),m(e,to,l),m(e,W,l),t(W,ye),t(ye,zs),v(Xe,zs,null),t(W,Gi),t(W,Ms),t(Ms,Ri),m(e,so,l),m(e,yt,l),t(yt,zi),m(e,ao,l),v(be,e,l),m(e,oo,l),m(e,we,l),t(we,Mi),t(we,bt),t(bt,Hi),t(we,Vi),m(e,no,l),v(Ye,e,l),m(e,io,l),m(e,wt,l),t(wt,Oi),m(e,ro,l),m(e,K,l),t(K,ge),t(ge,Hs),v(We,Hs,null),t(K,Bi),t(K,Vs),t(Vs,Li),m(e,lo,l),m(e,gt,l),t(gt,Xi),m(e,po,l),m(e,I,l),t(I,Yi),t(I,$t),t($t,Wi),t(I,Ki),t(I,Et),t(Et,Qi),t(I,Ji),m(e,fo,l),m(e,$e,l),t($e,Zi),t($e,jt),t(jt,er),t($e,tr),m(e,mo,l),v(Ke,e,l),m(e,ho,l),m(e,kt,l),t(kt,sr),m(e,co,l),m(e,Tt,l),m(e,uo,l),m(e,Q,l),t(Q,Ee),t(Ee,Os),v(Qe,Os,null),t(Q,ar),t(Q,Bs),t(Bs,or),m(e,_o,l),m(e,Pt,l),t(Pt,nr),m(e,vo,l),m(e,je,l),t(je,ir),t(je,Dt),t(Dt,rr),t(je,lr),m(e,yo,l),v(Je,e,l),m(e,bo,l),m(e,At,l),t(At,pr),m(e,wo,l),m(e,ke,l),t(ke,fr),t(ke,Ls),t(Ls,mr),t(ke,hr),m(e,go,l),v(Te,e,l),m(e,$o,l),m(e,St,l),t(St,cr),m(e,Eo,l),v(Ze,e,l),m(e,jo,l),m(e,D,l),t(D,Xs),t(Xs,ur),t(D,dr),t(D,Ys),t(Ys,_r),t(D,vr),t(D,Ws),t(Ws,yr),t(D,br),t(D,et),t(et,wr),t(D,gr),m(e,ko,l),m(e,qt,l),m(e,To,l),m(e,J,l),t(J,Pe),t(Pe,Ks),v(tt,Ks,null),t(J,$r),t(J,Qs),t(Qs,Er),m(e,Po,l),m(e,st,l),t(st,Nt),t(Nt,jr),t(st,kr),m(e,Do,l),m(e,C,l),t(C,Tr),t(C,Js),t(Js,Pr),t(C,Dr),t(C,Zs),t(Zs,Ar),t(C,Sr),m(e,Ao,l),m(e,xt,l),t(xt,qr),m(e,So,l),m(e,De,l),t(De,Nr),t(De,ea),t(ea,xr),t(De,Ur),m(e,qo,l),v(at,e,l),m(e,No,l),m(e,Ut,l),t(Ut,Ir),m(e,xo,l),v(ot,e,l),m(e,Uo,l),v(Ae,e,l),m(e,Io,l),m(e,Z,l),t(Z,Se),t(Se,ta),v(nt,ta,null),t(Z,Cr),t(Z,sa),t(sa,Fr),m(e,Co,l),m(e,It,l),t(It,Gr),m(e,Fo,l),m(e,Ct,l),t(Ct,Rr),m(e,Go,l),v(it,e,l),m(e,Ro,l),m(e,ee,l),t(ee,qe),t(qe,aa),v(rt,aa,null),t(ee,zr),t(ee,oa),t(oa,Mr),m(e,zo,l),m(e,Ne,l),t(Ne,Hr),t(Ne,na),t(na,Vr),t(Ne,Or),m(e,Mo,l),m(e,Ft,l),t(Ft,Br),m(e,Ho,l),v(lt,e,l),m(e,Vo,l),m(e,xe,l),t(xe,Lr),t(xe,ia),t(ia,Xr),t(xe,Yr),m(e,Oo,l),v(pt,e,l),m(e,Bo,l),m(e,te,l),t(te,Ue),t(Ue,ra),v(ft,ra,null),t(te,Wr),t(te,la),t(la,Kr),m(e,Lo,l),m(e,F,l),t(F,Qr),t(F,mt),t(mt,Jr),t(F,Zr),t(F,ht),t(ht,el),t(F,tl),m(e,Xo,l),m(e,Gt,l),t(Gt,sl),m(e,Yo,l),m(e,Ie,l),t(Ie,pa),t(pa,se),t(se,fa),t(fa,al),t(se,ol),t(se,ma),t(ma,nl),t(se,il),t(se,ha),t(ha,rl),t(Ie,ll),t(Ie,E),t(E,ae),t(ae,ca),t(ca,pl),t(ae,fl),t(ae,ua),t(ua,ml),t(ae,hl),t(ae,da),t(da,cl),t(E,ul),t(E,oe),t(oe,_a),t(_a,dl),t(oe,_l),t(oe,va),t(va,vl),t(oe,yl),t(oe,ya),t(ya,bl),t(E,wl),t(E,ne),t(ne,ba),t(ba,gl),t(ne,$l),t(ne,wa),t(wa,El),t(ne,jl),t(ne,ga),t(ga,kl),t(E,Tl),t(E,ie),t(ie,$a),t($a,Pl),t(ie,Dl),t(ie,Ea),t(Ea,Al),t(ie,Sl),t(ie,ja),t(ja,ql),t(E,Nl),t(E,re),t(re,ka),t(ka,xl),t(re,Ul),t(re,Ta),t(Ta,Il),t(re,Cl),t(re,Pa),t(Pa,Fl),t(E,Gl),t(E,le),t(le,Da),t(Da,Rl),t(le,zl),t(le,Aa),t(Aa,Ml),t(le,Hl),t(le,Sa),t(Sa,Vl),t(E,Ol),t(E,pe),t(pe,qa),t(qa,Bl),t(pe,Ll),t(pe,Na),t(Na,Xl),t(pe,Yl),t(pe,xa),t(xa,Wl),m(e,Wo,l),m(e,Rt,l),t(Rt,Kl),m(e,Ko,l),m(e,G,l),t(G,Ua),t(Ua,Ql),t(G,Jl),t(G,Ia),t(Ia,Zl),t(G,ep),t(G,zt),t(zt,Mt),t(Mt,tp),t(zt,sp),m(e,Qo,l),v(ct,e,l),Jo=!0},p(e,[l]){const ut={};l&2&&(ut.$$scope={dirty:l,ctx:e}),ve.$set(ut);const Ca={};l&2&&(Ca.$$scope={dirty:l,ctx:e}),be.$set(Ca);const Fa={};l&2&&(Fa.$$scope={dirty:l,ctx:e}),Te.$set(Fa);const Ga={};l&2&&(Ga.$$scope={dirty:l,ctx:e}),Ae.$set(Ga)},i(e){Jo||(y(k.$$.fragment,e),y(Ge.$$.fragment,e),y(Me.$$.fragment,e),y(He.$$.fragment,e),y(Oe.$$.fragment,e),y(Be.$$.fragment,e),y(Le.$$.fragment,e),y(ve.$$.fragment,e),y(Xe.$$.fragment,e),y(be.$$.fragment,e),y(Ye.$$.fragment,e),y(We.$$.fragment,e),y(Ke.$$.fragment,e),y(Qe.$$.fragment,e),y(Je.$$.fragment,e),y(Te.$$.fragment,e),y(Ze.$$.fragment,e),y(tt.$$.fragment,e),y(at.$$.fragment,e),y(ot.$$.fragment,e),y(Ae.$$.fragment,e),y(nt.$$.fragment,e),y(it.$$.fragment,e),y(rt.$$.fragment,e),y(lt.$$.fragment,e),y(pt.$$.fragment,e),y(ft.$$.fragment,e),y(ct.$$.fragment,e),Jo=!0)},o(e){b(k.$$.fragment,e),b(Ge.$$.fragment,e),b(Me.$$.fragment,e),b(He.$$.fragment,e),b(Oe.$$.fragment,e),b(Be.$$.fragment,e),b(Le.$$.fragment,e),b(ve.$$.fragment,e),b(Xe.$$.fragment,e),b(be.$$.fragment,e),b(Ye.$$.fragment,e),b(We.$$.fragment,e),b(Ke.$$.fragment,e),b(Qe.$$.fragment,e),b(Je.$$.fragment,e),b(Te.$$.fragment,e),b(Ze.$$.fragment,e),b(tt.$$.fragment,e),b(at.$$.fragment,e),b(ot.$$.fragment,e),b(Ae.$$.fragment,e),b(nt.$$.fragment,e),b(it.$$.fragment,e),b(rt.$$.fragment,e),b(lt.$$.fragment,e),b(pt.$$.fragment,e),b(ft.$$.fragment,e),b(ct.$$.fragment,e),Jo=!1},d(e){s(u),e&&s(c),e&&s(g),w(k),e&&s(Ra),e&&s(S),e&&s(za),e&&s(_t),e&&s(Ma),e&&s(me),e&&s(Va),e&&s(vt),e&&s(Oa),e&&s(L),w(Ge),e&&s(Ba),e&&s(Re),e&&s(La),e&&s(ce),e&&s(Xa),w(Me,e),e&&s(Ya),e&&s(X),w(He),e&&s(Wa),e&&s(de),e&&s(Ka),w(Oe,e),e&&s(Qa),e&&s(Y),w(Be),e&&s(Ja),e&&s(U),e&&s(Za),w(Le,e),e&&s(eo),w(ve,e),e&&s(to),e&&s(W),w(Xe),e&&s(so),e&&s(yt),e&&s(ao),w(be,e),e&&s(oo),e&&s(we),e&&s(no),w(Ye,e),e&&s(io),e&&s(wt),e&&s(ro),e&&s(K),w(We),e&&s(lo),e&&s(gt),e&&s(po),e&&s(I),e&&s(fo),e&&s($e),e&&s(mo),w(Ke,e),e&&s(ho),e&&s(kt),e&&s(co),e&&s(Tt),e&&s(uo),e&&s(Q),w(Qe),e&&s(_o),e&&s(Pt),e&&s(vo),e&&s(je),e&&s(yo),w(Je,e),e&&s(bo),e&&s(At),e&&s(wo),e&&s(ke),e&&s(go),w(Te,e),e&&s($o),e&&s(St),e&&s(Eo),w(Ze,e),e&&s(jo),e&&s(D),e&&s(ko),e&&s(qt),e&&s(To),e&&s(J),w(tt),e&&s(Po),e&&s(st),e&&s(Do),e&&s(C),e&&s(Ao),e&&s(xt),e&&s(So),e&&s(De),e&&s(qo),w(at,e),e&&s(No),e&&s(Ut),e&&s(xo),w(ot,e),e&&s(Uo),w(Ae,e),e&&s(Io),e&&s(Z),w(nt),e&&s(Co),e&&s(It),e&&s(Fo),e&&s(Ct),e&&s(Go),w(it,e),e&&s(Ro),e&&s(ee),w(rt),e&&s(zo),e&&s(Ne),e&&s(Mo),e&&s(Ft),e&&s(Ho),w(lt,e),e&&s(Vo),e&&s(xe),e&&s(Oo),w(pt,e),e&&s(Bo),e&&s(te),w(ft),e&&s(Lo),e&&s(F),e&&s(Xo),e&&s(Gt),e&&s(Yo),e&&s(Ie),e&&s(Wo),e&&s(Rt),e&&s(Ko),e&&s(G),e&&s(Qo),w(ct,e)}}}const Sm={local:"memory-and-speed",sections:[{local:"enable-cudnn-autotuner",sections:[{local:"use-tf32-instead-of-fp32-on-ampere-and-later-cuda-devices",title:"Use tf32 instead of fp32 (on Ampere and later CUDA devices)"}],title:"Enable cuDNN auto-tuner"},{local:"half-precision-weights",title:"Half precision weights"},{local:"sliced-attention-for-additional-memory-savings",title:"Sliced attention for additional memory savings"},{local:"sliced-vae-decode-for-larger-batches",title:"Sliced VAE decode for larger batches"},{local:"offloading-to-cpu-with-accelerate-for-memory-savings",title:"Offloading to CPU with accelerate for memory savings"},{local:"model-offloading-for-fast-inference-and-memory-savings",title:"Model offloading for fast inference and memory savings"},{local:"using-channels-last-memory-format",title:"Using Channels Last memory format"},{local:"tracing",title:"Tracing"},{local:"memory-efficient-attention",title:"Memory Efficient Attention"}],title:"Memory and speed"};function qm(x){return jm(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Cm extends wm{constructor(u){super();gm(this,u,qm,Am,$m,{})}}export{Cm as default,Sm as metadata};
