import{S as gl,i as _l,s as wl,e as a,k as u,w as v,t as l,M as vl,c as i,d as t,m as h,a as n,x as y,h as r,b as p,N as yl,G as o,g as f,y as b,q as E,o as A,B as $,v as bl}from"../../chunks/vendor-hf-doc-builder.js";import{T as ml}from"../../chunks/Tip-hf-doc-builder.js";import{I as Gt}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as Yo}from"../../chunks/CodeBlock-hf-doc-builder.js";function El(je){let d,g;return{c(){d=a("p"),g=l(`LoRA allows us to achieve greater memory efficiency since the pretrained weights are kept frozen and only the LoRA weights are trained, thereby
allowing us to run fine-tuning on consumer GPUs like Tesla T4, RTX 3080 or even RTX 2080 Ti! One can get access to GPUs like T4 in the free
tiers of Kaggle Kernels and Google Colab Notebooks.`)},l(c){d=i(c,"P",{});var m=n(d);g=r(m,`LoRA allows us to achieve greater memory efficiency since the pretrained weights are kept frozen and only the LoRA weights are trained, thereby
allowing us to run fine-tuning on consumer GPUs like Tesla T4, RTX 3080 or even RTX 2080 Ti! One can get access to GPUs like T4 in the free
tiers of Kaggle Kernels and Google Colab Notebooks.`),m.forEach(t)},m(c,m){f(c,d,m),o(d,g)},d(c){c&&t(d)}}}function Al(je){let d,g;return{c(){d=a("p"),g=l("When using LoRA we can use a much higher learning rate (typically 1e-4 as opposed to ~1e-6) compared to non-LoRA Dreambooth fine-tuning.")},l(c){d=i(c,"P",{});var m=n(d);g=r(m,"When using LoRA we can use a much higher learning rate (typically 1e-4 as opposed to ~1e-6) compared to non-LoRA Dreambooth fine-tuning."),m.forEach(t)},m(c,m){f(c,d,m),o(d,g)},d(c){c&&t(d)}}}function $l(je){let d,g,c,m,Fe,Q,Vo,Qe,Ko,zt,Te,Xo,Bt,k,Zo,ee,Jo,Fo,et,Qo,ea,Wt,L,ta,tt,oa,aa,ot,ia,na,Ht,R,te,la,oe,ra,sa,fa,at,pa,ua,ae,ha,it,da,ca,Yt,qe,nt,lt,ma,Vt,S,ie,ga,_a,ne,wa,va,Kt,N,Xt,j,O,rt,le,ya,st,ba,Zt,Ie,Ea,Jt,D,ft,re,Aa,$a,pt,se,ka,La,ut,fe,Ra,Ft,Ce,Da,Qt,M,ht,pe,Pa,xa,dt,ue,Sa,eo,Ne,ja,to,he,oo,U,Ta,ct,qa,Ia,ao,Oe,Ca,io,G,no,Me,Na,lo,z,Ue,Oa,de,mt,Ma,Ua,gt,Ga,za,Ge,Ba,ce,_t,Wa,Ha,wt,Ya,ro,B,Va,me,Ka,Xa,so,W,Za,ge,Ja,Fa,fo,T,H,vt,_e,Qa,yt,ei,po,P,ti,bt,oi,ai,we,ii,ni,uo,ve,ho,ze,li,co,Be,Xi,mo,q,ye,Et,ri,si,be,fi,pi,go,x,ui,Ee,At,hi,di,Ae,$t,ci,mi,_o,$e,wo,Y,gi,kt,_i,wi,vo,V,vi,Lt,yi,bi,yo,K,Ei,ke,Ai,$i,bo,I,X,Rt,Le,ki,Dt,Li,Eo,We,Ri,Ao,_,Di,Pt,Pi,xi,xt,Si,ji,St,Ti,qi,$o,Re,ko,w,Ii,jt,Ci,Ni,Tt,Oi,Mi,qt,Ui,Gi,Lo,C,Z,It,De,zi,Ct,Bi,Ro,He,Pe,Wi,xe,Nt,Hi,Yi,Do;return Q=new Gt({}),N=new ml({props:{$$slots:{default:[El]},$$scope:{ctx:je}}}),le=new Gt({}),he=new Yo({props:{code:`export MODEL_NAME="runwayml/stable-diffusion-v1-5"
export INSTANCE_DIR="path-to-instance-images"
export OUTPUT_DIR="path-to-save-model"

accelerate launch train_dreambooth_lora.py \\
  --pretrained_model_name_or_path=$MODEL_NAME  \\
  --instance_data_dir=$INSTANCE_DIR \\
  --output_dir=$OUTPUT_DIR \\
  --instance_prompt="a photo of sks dog" \\
  --resolution=512 \\
  --train_batch_size=1 \\
  --gradient_accumulation_steps=1 \\
  --checkpointing_steps=100 \\
  --learning_rate=1e-4 \\
  --report_to="wandb" \\
  --lr_scheduler="constant" \\
  --lr_warmup_steps=0 \\
  --max_train_steps=500 \\
  --validation_prompt="A photo of sks dog in a bucket" \\
  --validation_epochs=50 \\
  --seed="0" \\
  --push_to_hub`,highlighted:`<span class="hljs-built_in">export</span> MODEL_NAME=<span class="hljs-string">&quot;runwayml/stable-diffusion-v1-5&quot;</span>
<span class="hljs-built_in">export</span> INSTANCE_DIR=<span class="hljs-string">&quot;path-to-instance-images&quot;</span>
<span class="hljs-built_in">export</span> OUTPUT_DIR=<span class="hljs-string">&quot;path-to-save-model&quot;</span>

accelerate launch train_dreambooth_lora.py \\
  --pretrained_model_name_or_path=<span class="hljs-variable">$MODEL_NAME</span>  \\
  --instance_data_dir=<span class="hljs-variable">$INSTANCE_DIR</span> \\
  --output_dir=<span class="hljs-variable">$OUTPUT_DIR</span> \\
  --instance_prompt=<span class="hljs-string">&quot;a photo of sks dog&quot;</span> \\
  --resolution=512 \\
  --train_batch_size=1 \\
  --gradient_accumulation_steps=1 \\
  --checkpointing_steps=100 \\
  --learning_rate=1e-4 \\
  --report_to=<span class="hljs-string">&quot;wandb&quot;</span> \\
  --lr_scheduler=<span class="hljs-string">&quot;constant&quot;</span> \\
  --lr_warmup_steps=0 \\
  --max_train_steps=500 \\
  --validation_prompt=<span class="hljs-string">&quot;A photo of sks dog in a bucket&quot;</span> \\
  --validation_epochs=50 \\
  --seed=<span class="hljs-string">&quot;0&quot;</span> \\
  --push_to_hub`}}),G=new ml({props:{$$slots:{default:[Al]},$$scope:{ctx:je}}}),_e=new Gt({}),ve=new Yo({props:{code:`from diffusers import StableDiffusionPipeline
import torch

model_path = "sayakpaul/sd-model-finetuned-lora-t4"
pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4", torch_dtype=torch.float16)
pipe.unet.load_attn_procs(model_path)
pipe.to("cuda")

prompt = "A pokemon with blue eyes."
image = pipe(prompt, num_inference_steps=30, guidance_scale=7.5).images[0]
image.save("pokemon.png")`,highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline
<span class="hljs-keyword">import</span> torch

model_path = <span class="hljs-string">&quot;sayakpaul/sd-model-finetuned-lora-t4&quot;</span>
pipe = StableDiffusionPipeline.from_pretrained(<span class="hljs-string">&quot;CompVis/stable-diffusion-v1-4&quot;</span>, torch_dtype=torch.float16)
pipe.unet.load_attn_procs(model_path)
pipe.to(<span class="hljs-string">&quot;cuda&quot;</span>)

prompt = <span class="hljs-string">&quot;A pokemon with blue eyes.&quot;</span>
image = pipe(prompt, num_inference_steps=<span class="hljs-number">30</span>, guidance_scale=<span class="hljs-number">7.5</span>).images[<span class="hljs-number">0</span>]
image.save(<span class="hljs-string">&quot;pokemon.png&quot;</span>)`}}),$e=new Yo({props:{code:`from huggingface_hub.repocard import RepoCard

card = RepoCard.load("sayakpaul/sd-model-finetuned-lora-t4")
base_model = card.data.to_dict()["base_model"]
# 'CompVis/stable-diffusion-v1-4'`,highlighted:`<span class="hljs-keyword">from</span> huggingface_hub.repocard <span class="hljs-keyword">import</span> RepoCard

card = RepoCard.load(<span class="hljs-string">&quot;sayakpaul/sd-model-finetuned-lora-t4&quot;</span>)
base_model = card.data.to_dict()[<span class="hljs-string">&quot;base_model&quot;</span>]
<span class="hljs-comment"># &#x27;CompVis/stable-diffusion-v1-4&#x27;</span>`}}),Le=new Gt({}),Re=new Yo({props:{code:`from diffusers import StableDiffusionPipeline
import torch

model_path = "sayakpaul/sd-model-finetuned-lora-t4"
pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4", torch_dtype=torch.float16)
pipe.unet.load_attn_procs(model_path)
pipe.to("cuda")

prompt = "A pokemon with blue eyes."
image = pipe(prompt, num_inference_steps=30, guidance_scale=7.5, cross_attention_kwargs={"scale": 0.5}).images[0]
image.save("pokemon.png")`,highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline
<span class="hljs-keyword">import</span> torch

model_path = <span class="hljs-string">&quot;sayakpaul/sd-model-finetuned-lora-t4&quot;</span>
pipe = StableDiffusionPipeline.from_pretrained(<span class="hljs-string">&quot;CompVis/stable-diffusion-v1-4&quot;</span>, torch_dtype=torch.float16)
pipe.unet.load_attn_procs(model_path)
pipe.to(<span class="hljs-string">&quot;cuda&quot;</span>)

prompt = <span class="hljs-string">&quot;A pokemon with blue eyes.&quot;</span>
image = pipe(prompt, num_inference_steps=<span class="hljs-number">30</span>, guidance_scale=<span class="hljs-number">7.5</span>, cross_attention_kwargs={<span class="hljs-string">&quot;scale&quot;</span>: <span class="hljs-number">0.5</span>}).images[<span class="hljs-number">0</span>]
image.save(<span class="hljs-string">&quot;pokemon.png&quot;</span>)`}}),De=new Gt({}),{c(){d=a("meta"),g=u(),c=a("h1"),m=a("a"),Fe=a("span"),v(Q.$$.fragment),Vo=u(),Qe=a("span"),Ko=l("LoRA Support in Diffusers"),zt=u(),Te=a("p"),Xo=l("Diffusers supports LoRA for faster fine-tuning of Stable Diffusion, allowing greater memory efficiency and easier portability."),Bt=u(),k=a("p"),Zo=l(`Low-Rank Adaption of Large Language Models was first introduced by Microsoft in
`),ee=a("a"),Jo=l("LoRA: Low-Rank Adaptation of Large Language Models"),Fo=l(" by "),et=a("em"),Qo=l("Edward J. Hu, Yelong Shen, Phillip Wallis, Zeyuan Allen-Zhu, Yuanzhi Li, Shean Wang, Lu Wang, Weizhu Chen"),ea=l("."),Wt=u(),L=a("p"),ta=l("In a nutshell, LoRA allows adapting pretrained models by adding pairs of rank-decomposition weight matrices (called "),tt=a("strong"),oa=l("update matrices"),aa=l(`)
to existing weights and `),ot=a("strong"),ia=l("only"),na=l(" training those newly added weights. This has a couple of advantages:"),Ht=u(),R=a("ul"),te=a("li"),la=l("Previous pretrained weights are kept frozen so that the model is not so prone to "),oe=a("a"),ra=l("catastrophic forgetting"),sa=l("."),fa=u(),at=a("li"),pa=l("Rank-decomposition matrices have significantly fewer parameters than the original model, which means that trained LoRA weights are easily portable."),ua=u(),ae=a("li"),ha=l("LoRA matrices are generally added to the attention layers of the original model and they control to which extent the model is adapted toward new training images via a "),it=a("code"),da=l("scale"),ca=l(" parameter."),Yt=u(),qe=a("p"),nt=a("strong"),lt=a("strong"),ma=l(`Note that the usage of LoRA is not just limited to attention layers. In the original LoRA work, the authors found out that just amending
the attention layers of a language model is sufficient to obtain good downstream performance with great efficiency. This is why, it\u2019s common
to just add the LoRA weights to the attention layers of a model.`),Vt=u(),S=a("p"),ie=a("a"),ga=l("cloneofsimo"),_a=l(" was the first to try out LoRA training for Stable Diffusion in the popular "),ne=a("a"),wa=l("lora"),va=l(" GitHub repository."),Kt=u(),v(N.$$.fragment),Xt=u(),j=a("h2"),O=a("a"),rt=a("span"),v(le.$$.fragment),ya=u(),st=a("span"),ba=l("Getting started with LoRA for fine-tuning"),Zt=u(),Ie=a("p"),Ea=l("Stable Diffusion can be fine-tuned in different ways:"),Jt=u(),D=a("ul"),ft=a("li"),re=a("a"),Aa=l("Textual inversion"),$a=u(),pt=a("li"),se=a("a"),ka=l("DreamBooth"),La=u(),ut=a("li"),fe=a("a"),Ra=l("Text2Image fine-tuning"),Ft=u(),Ce=a("p"),Da=l("We provide two end-to-end examples that show how to run fine-tuning with LoRA:"),Qt=u(),M=a("ul"),ht=a("li"),pe=a("a"),Pa=l("DreamBooth"),xa=u(),dt=a("li"),ue=a("a"),Sa=l("Text2Image"),eo=u(),Ne=a("p"),ja=l("If you want to perform DreamBooth training with LoRA, for instance, you would run:"),to=u(),v(he.$$.fragment),oo=u(),U=a("p"),Ta=l(`A similar process can be followed to fully fine-tune Stable Diffusion on a custom dataset using the
`),ct=a("code"),qa=l("examples/text_to_image/train_text_to_image_lora.py"),Ia=l(" script."),ao=u(),Oe=a("p"),Ca=l("Refer to the respective examples linked above to learn more."),io=u(),v(G.$$.fragment),no=u(),Me=a("p"),Na=l(`But there is no free lunch. For the given dataset and expected generation quality, you\u2019d still need to experiment with
different hyperparameters. Here are some important ones:`),lo=u(),z=a("ul"),Ue=a("li"),Oa=l("Training time"),de=a("ul"),mt=a("li"),Ma=l("Learning rate"),Ua=u(),gt=a("li"),Ga=l("Number of training steps"),za=u(),Ge=a("li"),Ba=l("Inference time "),ce=a("ul"),_t=a("li"),Wa=l("Number of steps"),Ha=u(),wt=a("li"),Ya=l("Scheduler type"),ro=u(),B=a("p"),Va=l("Additionally, you can follow "),me=a("a"),Ka=l("this blog"),Xa=l(` that documents some of our experimental
findings for performing DreamBooth training of Stable Diffusion.`),so=u(),W=a("p"),Za=l(`When fine-tuning, the LoRA update matrices are only added to the attention layers. To enable this, we added new weight
loading functionalities. Their details are available `),ge=a("a"),Ja=l("here"),Fa=l("."),fo=u(),T=a("h2"),H=a("a"),vt=a("span"),v(_e.$$.fragment),Qa=u(),yt=a("span"),ei=l("Inference"),po=u(),P=a("p"),ti=l("Assuming you used the "),bt=a("code"),oi=l("examples/text_to_image/train_text_to_image_lora.py"),ai=l(" to fine-tune Stable Diffusion on the "),we=a("a"),ii=l(`Pokemon
dataset`),ni=l(", you can perform inference like so:"),uo=u(),v(ve.$$.fragment),ho=u(),ze=a("p"),li=l("Here are some example images you can expect:"),co=u(),Be=a("img"),mo=u(),q=a("p"),ye=a("a"),Et=a("code"),ri=l("sayakpaul/sd-model-finetuned-lora-t4"),si=l(" contains "),be=a("a"),fi=l("LoRA fine-tuned update matrices"),pi=l(`
which is only 3 MBs in size. During inference, the pre-trained Stable Diffusion checkpoints are loaded alongside these update
matrices and then they are combined to run inference.`),go=u(),x=a("p"),ui=l("You can use the "),Ee=a("a"),At=a("code"),hi=l("huggingface_hub"),di=l(` library to retrieve the base model
from `),Ae=a("a"),$t=a("code"),ci=l("sayakpaul/sd-model-finetuned-lora-t4"),mi=l(" like so:"),_o=u(),v($e.$$.fragment),wo=u(),Y=a("p"),gi=l("And then you can use "),kt=a("code"),_i=l("pipe = StableDiffusionPipeline.from_pretrained(base_model, torch_dtype=torch.float16)"),wi=l("."),vo=u(),V=a("p"),vi=l("This is especially useful when you don\u2019t want to hardcode the base model identifier during initializing the "),Lt=a("code"),yi=l("StableDiffusionPipeline"),bi=l("."),yo=u(),K=a("p"),Ei=l(`Inference for DreamBooth training remains the same. Check
`),ke=a("a"),Ai=l("this section"),$i=l(" for more details."),bo=u(),I=a("h3"),X=a("a"),Rt=a("span"),v(Le.$$.fragment),ki=u(),Dt=a("span"),Li=l("Merging LoRA with original model"),Eo=u(),We=a("p"),Ri=l("When performing inference, you can merge the trained LoRA weights with the frozen pre-trained model weights, to interpolate between the original model\u2019s inference result (as if no fine-tuning had occurred) and the fully fine-tuned version."),Ao=u(),_=a("p"),Di=l("You can adjust the merging ratio with a parameter called \u03B1 (alpha) in the paper, or "),Pt=a("code"),Pi=l("scale"),xi=l(" in our implementation. You can tweak it with the following code, that passes "),xt=a("code"),Si=l("scale"),ji=l(" as "),St=a("code"),Ti=l("cross_attention_kwargs"),qi=l(" in the pipeline call:"),$o=u(),v(Re.$$.fragment),ko=u(),w=a("p"),Ii=l("A value of "),jt=a("code"),Ci=l("0"),Ni=l(" is the same as "),Tt=a("em"),Oi=l("not"),Mi=l(" using the LoRA weights, whereas "),qt=a("code"),Ui=l("1"),Gi=l(" means only the LoRA fine-tuned weights will be used. Values between 0 and 1 will interpolate between the two versions."),Lo=u(),C=a("h2"),Z=a("a"),It=a("span"),v(De.$$.fragment),zi=u(),Ct=a("span"),Bi=l("Known limitations"),Ro=u(),He=a("ul"),Pe=a("li"),Wi=l("Currently, we only support LoRA for the attention layers of "),xe=a("a"),Nt=a("code"),Hi=l("UNet2DConditionModel"),Yi=l("."),this.h()},l(e){const s=vl('[data-svelte="svelte-1phssyn"]',document.head);d=i(s,"META",{name:!0,content:!0}),s.forEach(t),g=h(e),c=i(e,"H1",{class:!0});var Se=n(c);m=i(Se,"A",{id:!0,class:!0,href:!0});var Ot=n(m);Fe=i(Ot,"SPAN",{});var Zi=n(Fe);y(Q.$$.fragment,Zi),Zi.forEach(t),Ot.forEach(t),Vo=h(Se),Qe=i(Se,"SPAN",{});var Ji=n(Qe);Ko=r(Ji,"LoRA Support in Diffusers"),Ji.forEach(t),Se.forEach(t),zt=h(e),Te=i(e,"P",{});var Fi=n(Te);Xo=r(Fi,"Diffusers supports LoRA for faster fine-tuning of Stable Diffusion, allowing greater memory efficiency and easier portability."),Fi.forEach(t),Bt=h(e),k=i(e,"P",{});var Ye=n(k);Zo=r(Ye,`Low-Rank Adaption of Large Language Models was first introduced by Microsoft in
`),ee=i(Ye,"A",{href:!0,rel:!0});var Qi=n(ee);Jo=r(Qi,"LoRA: Low-Rank Adaptation of Large Language Models"),Qi.forEach(t),Fo=r(Ye," by "),et=i(Ye,"EM",{});var en=n(et);Qo=r(en,"Edward J. Hu, Yelong Shen, Phillip Wallis, Zeyuan Allen-Zhu, Yuanzhi Li, Shean Wang, Lu Wang, Weizhu Chen"),en.forEach(t),ea=r(Ye,"."),Ye.forEach(t),Wt=h(e),L=i(e,"P",{});var Ve=n(L);ta=r(Ve,"In a nutshell, LoRA allows adapting pretrained models by adding pairs of rank-decomposition weight matrices (called "),tt=i(Ve,"STRONG",{});var tn=n(tt);oa=r(tn,"update matrices"),tn.forEach(t),aa=r(Ve,`)
to existing weights and `),ot=i(Ve,"STRONG",{});var on=n(ot);ia=r(on,"only"),on.forEach(t),na=r(Ve," training those newly added weights. This has a couple of advantages:"),Ve.forEach(t),Ht=h(e),R=i(e,"UL",{});var Ke=n(R);te=i(Ke,"LI",{});var Po=n(te);la=r(Po,"Previous pretrained weights are kept frozen so that the model is not so prone to "),oe=i(Po,"A",{href:!0,rel:!0});var an=n(oe);ra=r(an,"catastrophic forgetting"),an.forEach(t),sa=r(Po,"."),Po.forEach(t),fa=h(Ke),at=i(Ke,"LI",{});var nn=n(at);pa=r(nn,"Rank-decomposition matrices have significantly fewer parameters than the original model, which means that trained LoRA weights are easily portable."),nn.forEach(t),ua=h(Ke),ae=i(Ke,"LI",{});var xo=n(ae);ha=r(xo,"LoRA matrices are generally added to the attention layers of the original model and they control to which extent the model is adapted toward new training images via a "),it=i(xo,"CODE",{});var ln=n(it);da=r(ln,"scale"),ln.forEach(t),ca=r(xo," parameter."),xo.forEach(t),Ke.forEach(t),Yt=h(e),qe=i(e,"P",{});var rn=n(qe);nt=i(rn,"STRONG",{});var sn=n(nt);lt=i(sn,"STRONG",{});var fn=n(lt);ma=r(fn,`Note that the usage of LoRA is not just limited to attention layers. In the original LoRA work, the authors found out that just amending
the attention layers of a language model is sufficient to obtain good downstream performance with great efficiency. This is why, it\u2019s common
to just add the LoRA weights to the attention layers of a model.`),fn.forEach(t),sn.forEach(t),rn.forEach(t),Vt=h(e),S=i(e,"P",{});var Mt=n(S);ie=i(Mt,"A",{href:!0,rel:!0});var pn=n(ie);ga=r(pn,"cloneofsimo"),pn.forEach(t),_a=r(Mt," was the first to try out LoRA training for Stable Diffusion in the popular "),ne=i(Mt,"A",{href:!0,rel:!0});var un=n(ne);wa=r(un,"lora"),un.forEach(t),va=r(Mt," GitHub repository."),Mt.forEach(t),Kt=h(e),y(N.$$.fragment,e),Xt=h(e),j=i(e,"H2",{class:!0});var So=n(j);O=i(So,"A",{id:!0,class:!0,href:!0});var hn=n(O);rt=i(hn,"SPAN",{});var dn=n(rt);y(le.$$.fragment,dn),dn.forEach(t),hn.forEach(t),ya=h(So),st=i(So,"SPAN",{});var cn=n(st);ba=r(cn,"Getting started with LoRA for fine-tuning"),cn.forEach(t),So.forEach(t),Zt=h(e),Ie=i(e,"P",{});var mn=n(Ie);Ea=r(mn,"Stable Diffusion can be fine-tuned in different ways:"),mn.forEach(t),Jt=h(e),D=i(e,"UL",{});var Xe=n(D);ft=i(Xe,"LI",{});var gn=n(ft);re=i(gn,"A",{href:!0,rel:!0});var _n=n(re);Aa=r(_n,"Textual inversion"),_n.forEach(t),gn.forEach(t),$a=h(Xe),pt=i(Xe,"LI",{});var wn=n(pt);se=i(wn,"A",{href:!0,rel:!0});var vn=n(se);ka=r(vn,"DreamBooth"),vn.forEach(t),wn.forEach(t),La=h(Xe),ut=i(Xe,"LI",{});var yn=n(ut);fe=i(yn,"A",{href:!0,rel:!0});var bn=n(fe);Ra=r(bn,"Text2Image fine-tuning"),bn.forEach(t),yn.forEach(t),Xe.forEach(t),Ft=h(e),Ce=i(e,"P",{});var En=n(Ce);Da=r(En,"We provide two end-to-end examples that show how to run fine-tuning with LoRA:"),En.forEach(t),Qt=h(e),M=i(e,"UL",{});var jo=n(M);ht=i(jo,"LI",{});var An=n(ht);pe=i(An,"A",{href:!0,rel:!0});var $n=n(pe);Pa=r($n,"DreamBooth"),$n.forEach(t),An.forEach(t),xa=h(jo),dt=i(jo,"LI",{});var kn=n(dt);ue=i(kn,"A",{href:!0,rel:!0});var Ln=n(ue);Sa=r(Ln,"Text2Image"),Ln.forEach(t),kn.forEach(t),jo.forEach(t),eo=h(e),Ne=i(e,"P",{});var Rn=n(Ne);ja=r(Rn,"If you want to perform DreamBooth training with LoRA, for instance, you would run:"),Rn.forEach(t),to=h(e),y(he.$$.fragment,e),oo=h(e),U=i(e,"P",{});var To=n(U);Ta=r(To,`A similar process can be followed to fully fine-tune Stable Diffusion on a custom dataset using the
`),ct=i(To,"CODE",{});var Dn=n(ct);qa=r(Dn,"examples/text_to_image/train_text_to_image_lora.py"),Dn.forEach(t),Ia=r(To," script."),To.forEach(t),ao=h(e),Oe=i(e,"P",{});var Pn=n(Oe);Ca=r(Pn,"Refer to the respective examples linked above to learn more."),Pn.forEach(t),io=h(e),y(G.$$.fragment,e),no=h(e),Me=i(e,"P",{});var xn=n(Me);Na=r(xn,`But there is no free lunch. For the given dataset and expected generation quality, you\u2019d still need to experiment with
different hyperparameters. Here are some important ones:`),xn.forEach(t),lo=h(e),z=i(e,"UL",{});var qo=n(z);Ue=i(qo,"LI",{});var Vi=n(Ue);Oa=r(Vi,"Training time"),de=i(Vi,"UL",{});var Io=n(de);mt=i(Io,"LI",{});var Sn=n(mt);Ma=r(Sn,"Learning rate"),Sn.forEach(t),Ua=h(Io),gt=i(Io,"LI",{});var jn=n(gt);Ga=r(jn,"Number of training steps"),jn.forEach(t),Io.forEach(t),Vi.forEach(t),za=h(qo),Ge=i(qo,"LI",{});var Ki=n(Ge);Ba=r(Ki,"Inference time "),ce=i(Ki,"UL",{});var Co=n(ce);_t=i(Co,"LI",{});var Tn=n(_t);Wa=r(Tn,"Number of steps"),Tn.forEach(t),Ha=h(Co),wt=i(Co,"LI",{});var qn=n(wt);Ya=r(qn,"Scheduler type"),qn.forEach(t),Co.forEach(t),Ki.forEach(t),qo.forEach(t),ro=h(e),B=i(e,"P",{});var No=n(B);Va=r(No,"Additionally, you can follow "),me=i(No,"A",{href:!0,rel:!0});var In=n(me);Ka=r(In,"this blog"),In.forEach(t),Xa=r(No,` that documents some of our experimental
findings for performing DreamBooth training of Stable Diffusion.`),No.forEach(t),so=h(e),W=i(e,"P",{});var Oo=n(W);Za=r(Oo,`When fine-tuning, the LoRA update matrices are only added to the attention layers. To enable this, we added new weight
loading functionalities. Their details are available `),ge=i(Oo,"A",{href:!0,rel:!0});var Cn=n(ge);Ja=r(Cn,"here"),Cn.forEach(t),Fa=r(Oo,"."),Oo.forEach(t),fo=h(e),T=i(e,"H2",{class:!0});var Mo=n(T);H=i(Mo,"A",{id:!0,class:!0,href:!0});var Nn=n(H);vt=i(Nn,"SPAN",{});var On=n(vt);y(_e.$$.fragment,On),On.forEach(t),Nn.forEach(t),Qa=h(Mo),yt=i(Mo,"SPAN",{});var Mn=n(yt);ei=r(Mn,"Inference"),Mn.forEach(t),Mo.forEach(t),po=h(e),P=i(e,"P",{});var Ze=n(P);ti=r(Ze,"Assuming you used the "),bt=i(Ze,"CODE",{});var Un=n(bt);oi=r(Un,"examples/text_to_image/train_text_to_image_lora.py"),Un.forEach(t),ai=r(Ze," to fine-tune Stable Diffusion on the "),we=i(Ze,"A",{href:!0,rel:!0});var Gn=n(we);ii=r(Gn,`Pokemon
dataset`),Gn.forEach(t),ni=r(Ze,", you can perform inference like so:"),Ze.forEach(t),uo=h(e),y(ve.$$.fragment,e),ho=h(e),ze=i(e,"P",{});var zn=n(ze);li=r(zn,"Here are some example images you can expect:"),zn.forEach(t),co=h(e),Be=i(e,"IMG",{src:!0}),mo=h(e),q=i(e,"P",{});var Ut=n(q);ye=i(Ut,"A",{href:!0,rel:!0});var Bn=n(ye);Et=i(Bn,"CODE",{});var Wn=n(Et);ri=r(Wn,"sayakpaul/sd-model-finetuned-lora-t4"),Wn.forEach(t),Bn.forEach(t),si=r(Ut," contains "),be=i(Ut,"A",{href:!0,rel:!0});var Hn=n(be);fi=r(Hn,"LoRA fine-tuned update matrices"),Hn.forEach(t),pi=r(Ut,`
which is only 3 MBs in size. During inference, the pre-trained Stable Diffusion checkpoints are loaded alongside these update
matrices and then they are combined to run inference.`),Ut.forEach(t),go=h(e),x=i(e,"P",{});var Je=n(x);ui=r(Je,"You can use the "),Ee=i(Je,"A",{href:!0,rel:!0});var Yn=n(Ee);At=i(Yn,"CODE",{});var Vn=n(At);hi=r(Vn,"huggingface_hub"),Vn.forEach(t),Yn.forEach(t),di=r(Je,` library to retrieve the base model
from `),Ae=i(Je,"A",{href:!0,rel:!0});var Kn=n(Ae);$t=i(Kn,"CODE",{});var Xn=n($t);ci=r(Xn,"sayakpaul/sd-model-finetuned-lora-t4"),Xn.forEach(t),Kn.forEach(t),mi=r(Je," like so:"),Je.forEach(t),_o=h(e),y($e.$$.fragment,e),wo=h(e),Y=i(e,"P",{});var Uo=n(Y);gi=r(Uo,"And then you can use "),kt=i(Uo,"CODE",{});var Zn=n(kt);_i=r(Zn,"pipe = StableDiffusionPipeline.from_pretrained(base_model, torch_dtype=torch.float16)"),Zn.forEach(t),wi=r(Uo,"."),Uo.forEach(t),vo=h(e),V=i(e,"P",{});var Go=n(V);vi=r(Go,"This is especially useful when you don\u2019t want to hardcode the base model identifier during initializing the "),Lt=i(Go,"CODE",{});var Jn=n(Lt);yi=r(Jn,"StableDiffusionPipeline"),Jn.forEach(t),bi=r(Go,"."),Go.forEach(t),yo=h(e),K=i(e,"P",{});var zo=n(K);Ei=r(zo,`Inference for DreamBooth training remains the same. Check
`),ke=i(zo,"A",{href:!0,rel:!0});var Fn=n(ke);Ai=r(Fn,"this section"),Fn.forEach(t),$i=r(zo," for more details."),zo.forEach(t),bo=h(e),I=i(e,"H3",{class:!0});var Bo=n(I);X=i(Bo,"A",{id:!0,class:!0,href:!0});var Qn=n(X);Rt=i(Qn,"SPAN",{});var el=n(Rt);y(Le.$$.fragment,el),el.forEach(t),Qn.forEach(t),ki=h(Bo),Dt=i(Bo,"SPAN",{});var tl=n(Dt);Li=r(tl,"Merging LoRA with original model"),tl.forEach(t),Bo.forEach(t),Eo=h(e),We=i(e,"P",{});var ol=n(We);Ri=r(ol,"When performing inference, you can merge the trained LoRA weights with the frozen pre-trained model weights, to interpolate between the original model\u2019s inference result (as if no fine-tuning had occurred) and the fully fine-tuned version."),ol.forEach(t),Ao=h(e),_=i(e,"P",{});var J=n(_);Di=r(J,"You can adjust the merging ratio with a parameter called \u03B1 (alpha) in the paper, or "),Pt=i(J,"CODE",{});var al=n(Pt);Pi=r(al,"scale"),al.forEach(t),xi=r(J," in our implementation. You can tweak it with the following code, that passes "),xt=i(J,"CODE",{});var il=n(xt);Si=r(il,"scale"),il.forEach(t),ji=r(J," as "),St=i(J,"CODE",{});var nl=n(St);Ti=r(nl,"cross_attention_kwargs"),nl.forEach(t),qi=r(J," in the pipeline call:"),J.forEach(t),$o=h(e),y(Re.$$.fragment,e),ko=h(e),w=i(e,"P",{});var F=n(w);Ii=r(F,"A value of "),jt=i(F,"CODE",{});var ll=n(jt);Ci=r(ll,"0"),ll.forEach(t),Ni=r(F," is the same as "),Tt=i(F,"EM",{});var rl=n(Tt);Oi=r(rl,"not"),rl.forEach(t),Mi=r(F," using the LoRA weights, whereas "),qt=i(F,"CODE",{});var sl=n(qt);Ui=r(sl,"1"),sl.forEach(t),Gi=r(F," means only the LoRA fine-tuned weights will be used. Values between 0 and 1 will interpolate between the two versions."),F.forEach(t),Lo=h(e),C=i(e,"H2",{class:!0});var Wo=n(C);Z=i(Wo,"A",{id:!0,class:!0,href:!0});var fl=n(Z);It=i(fl,"SPAN",{});var pl=n(It);y(De.$$.fragment,pl),pl.forEach(t),fl.forEach(t),zi=h(Wo),Ct=i(Wo,"SPAN",{});var ul=n(Ct);Bi=r(ul,"Known limitations"),ul.forEach(t),Wo.forEach(t),Ro=h(e),He=i(e,"UL",{});var hl=n(He);Pe=i(hl,"LI",{});var Ho=n(Pe);Wi=r(Ho,"Currently, we only support LoRA for the attention layers of "),xe=i(Ho,"A",{href:!0,rel:!0});var dl=n(xe);Nt=i(dl,"CODE",{});var cl=n(Nt);Hi=r(cl,"UNet2DConditionModel"),cl.forEach(t),dl.forEach(t),Yi=r(Ho,"."),Ho.forEach(t),hl.forEach(t),this.h()},h(){p(d,"name","hf:doc:metadata"),p(d,"content",JSON.stringify(kl)),p(m,"id","lora-support-in-diffusers"),p(m,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(m,"href","#lora-support-in-diffusers"),p(c,"class","relative group"),p(ee,"href","https://arxiv.org/abs/2106.09685"),p(ee,"rel","nofollow"),p(oe,"href","https://www.pnas.org/doi/10.1073/pnas.1611835114"),p(oe,"rel","nofollow"),p(ie,"href","https://github.com/cloneofsimo"),p(ie,"rel","nofollow"),p(ne,"href","https://github.com/cloneofsimo/lora"),p(ne,"rel","nofollow"),p(O,"id","getting-started-with-lora-for-finetuning"),p(O,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(O,"href","#getting-started-with-lora-for-finetuning"),p(j,"class","relative group"),p(re,"href","https://huggingface.co/docs/diffusers/main/en/training/text_inversion"),p(re,"rel","nofollow"),p(se,"href","https://huggingface.co/docs/diffusers/main/en/training/dreambooth"),p(se,"rel","nofollow"),p(fe,"href","https://huggingface.co/docs/diffusers/main/en/training/text2image"),p(fe,"rel","nofollow"),p(pe,"href","https://github.com/huggingface/diffusers/tree/main/examples/dreambooth#training-with-low-rank-adaptation-of-large-language-models-lora"),p(pe,"rel","nofollow"),p(ue,"href","https://github.com/huggingface/diffusers/tree/main/examples/text_to_image#training-with-lora"),p(ue,"rel","nofollow"),p(me,"href","https://huggingface.co/blog/dreambooth"),p(me,"rel","nofollow"),p(ge,"href","https://huggingface.co/docs/diffusers/main/en/api/loaders"),p(ge,"rel","nofollow"),p(H,"id","inference"),p(H,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(H,"href","#inference"),p(T,"class","relative group"),p(we,"href","https://huggingface.co/datasets/lambdalabs/pokemon-blip-captions"),p(we,"rel","nofollow"),yl(Be.src,Xi="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pokemon-collage.png")||p(Be,"src",Xi),p(ye,"href","https://huggingface.co/sayakpaul/sd-model-finetuned-lora-t4"),p(ye,"rel","nofollow"),p(be,"href","https://huggingface.co/sayakpaul/sd-model-finetuned-lora-t4/blob/main/pytorch_lora_weights.bin"),p(be,"rel","nofollow"),p(Ee,"href","https://github.com/huggingface/huggingface_hub"),p(Ee,"rel","nofollow"),p(Ae,"href","https://huggingface.co/sayakpaul/sd-model-finetuned-lora-t4"),p(Ae,"rel","nofollow"),p(ke,"href","https://github.com/huggingface/diffusers/tree/main/examples/dreambooth#inference-1"),p(ke,"rel","nofollow"),p(X,"id","merging-lora-with-original-model"),p(X,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(X,"href","#merging-lora-with-original-model"),p(I,"class","relative group"),p(Z,"id","known-limitations"),p(Z,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(Z,"href","#known-limitations"),p(C,"class","relative group"),p(xe,"href","https://huggingface.co/docs/diffusers/main/en/api/models#diffusers.UNet2DConditionModel"),p(xe,"rel","nofollow")},m(e,s){o(document.head,d),f(e,g,s),f(e,c,s),o(c,m),o(m,Fe),b(Q,Fe,null),o(c,Vo),o(c,Qe),o(Qe,Ko),f(e,zt,s),f(e,Te,s),o(Te,Xo),f(e,Bt,s),f(e,k,s),o(k,Zo),o(k,ee),o(ee,Jo),o(k,Fo),o(k,et),o(et,Qo),o(k,ea),f(e,Wt,s),f(e,L,s),o(L,ta),o(L,tt),o(tt,oa),o(L,aa),o(L,ot),o(ot,ia),o(L,na),f(e,Ht,s),f(e,R,s),o(R,te),o(te,la),o(te,oe),o(oe,ra),o(te,sa),o(R,fa),o(R,at),o(at,pa),o(R,ua),o(R,ae),o(ae,ha),o(ae,it),o(it,da),o(ae,ca),f(e,Yt,s),f(e,qe,s),o(qe,nt),o(nt,lt),o(lt,ma),f(e,Vt,s),f(e,S,s),o(S,ie),o(ie,ga),o(S,_a),o(S,ne),o(ne,wa),o(S,va),f(e,Kt,s),b(N,e,s),f(e,Xt,s),f(e,j,s),o(j,O),o(O,rt),b(le,rt,null),o(j,ya),o(j,st),o(st,ba),f(e,Zt,s),f(e,Ie,s),o(Ie,Ea),f(e,Jt,s),f(e,D,s),o(D,ft),o(ft,re),o(re,Aa),o(D,$a),o(D,pt),o(pt,se),o(se,ka),o(D,La),o(D,ut),o(ut,fe),o(fe,Ra),f(e,Ft,s),f(e,Ce,s),o(Ce,Da),f(e,Qt,s),f(e,M,s),o(M,ht),o(ht,pe),o(pe,Pa),o(M,xa),o(M,dt),o(dt,ue),o(ue,Sa),f(e,eo,s),f(e,Ne,s),o(Ne,ja),f(e,to,s),b(he,e,s),f(e,oo,s),f(e,U,s),o(U,Ta),o(U,ct),o(ct,qa),o(U,Ia),f(e,ao,s),f(e,Oe,s),o(Oe,Ca),f(e,io,s),b(G,e,s),f(e,no,s),f(e,Me,s),o(Me,Na),f(e,lo,s),f(e,z,s),o(z,Ue),o(Ue,Oa),o(Ue,de),o(de,mt),o(mt,Ma),o(de,Ua),o(de,gt),o(gt,Ga),o(z,za),o(z,Ge),o(Ge,Ba),o(Ge,ce),o(ce,_t),o(_t,Wa),o(ce,Ha),o(ce,wt),o(wt,Ya),f(e,ro,s),f(e,B,s),o(B,Va),o(B,me),o(me,Ka),o(B,Xa),f(e,so,s),f(e,W,s),o(W,Za),o(W,ge),o(ge,Ja),o(W,Fa),f(e,fo,s),f(e,T,s),o(T,H),o(H,vt),b(_e,vt,null),o(T,Qa),o(T,yt),o(yt,ei),f(e,po,s),f(e,P,s),o(P,ti),o(P,bt),o(bt,oi),o(P,ai),o(P,we),o(we,ii),o(P,ni),f(e,uo,s),b(ve,e,s),f(e,ho,s),f(e,ze,s),o(ze,li),f(e,co,s),f(e,Be,s),f(e,mo,s),f(e,q,s),o(q,ye),o(ye,Et),o(Et,ri),o(q,si),o(q,be),o(be,fi),o(q,pi),f(e,go,s),f(e,x,s),o(x,ui),o(x,Ee),o(Ee,At),o(At,hi),o(x,di),o(x,Ae),o(Ae,$t),o($t,ci),o(x,mi),f(e,_o,s),b($e,e,s),f(e,wo,s),f(e,Y,s),o(Y,gi),o(Y,kt),o(kt,_i),o(Y,wi),f(e,vo,s),f(e,V,s),o(V,vi),o(V,Lt),o(Lt,yi),o(V,bi),f(e,yo,s),f(e,K,s),o(K,Ei),o(K,ke),o(ke,Ai),o(K,$i),f(e,bo,s),f(e,I,s),o(I,X),o(X,Rt),b(Le,Rt,null),o(I,ki),o(I,Dt),o(Dt,Li),f(e,Eo,s),f(e,We,s),o(We,Ri),f(e,Ao,s),f(e,_,s),o(_,Di),o(_,Pt),o(Pt,Pi),o(_,xi),o(_,xt),o(xt,Si),o(_,ji),o(_,St),o(St,Ti),o(_,qi),f(e,$o,s),b(Re,e,s),f(e,ko,s),f(e,w,s),o(w,Ii),o(w,jt),o(jt,Ci),o(w,Ni),o(w,Tt),o(Tt,Oi),o(w,Mi),o(w,qt),o(qt,Ui),o(w,Gi),f(e,Lo,s),f(e,C,s),o(C,Z),o(Z,It),b(De,It,null),o(C,zi),o(C,Ct),o(Ct,Bi),f(e,Ro,s),f(e,He,s),o(He,Pe),o(Pe,Wi),o(Pe,xe),o(xe,Nt),o(Nt,Hi),o(Pe,Yi),Do=!0},p(e,[s]){const Se={};s&2&&(Se.$$scope={dirty:s,ctx:e}),N.$set(Se);const Ot={};s&2&&(Ot.$$scope={dirty:s,ctx:e}),G.$set(Ot)},i(e){Do||(E(Q.$$.fragment,e),E(N.$$.fragment,e),E(le.$$.fragment,e),E(he.$$.fragment,e),E(G.$$.fragment,e),E(_e.$$.fragment,e),E(ve.$$.fragment,e),E($e.$$.fragment,e),E(Le.$$.fragment,e),E(Re.$$.fragment,e),E(De.$$.fragment,e),Do=!0)},o(e){A(Q.$$.fragment,e),A(N.$$.fragment,e),A(le.$$.fragment,e),A(he.$$.fragment,e),A(G.$$.fragment,e),A(_e.$$.fragment,e),A(ve.$$.fragment,e),A($e.$$.fragment,e),A(Le.$$.fragment,e),A(Re.$$.fragment,e),A(De.$$.fragment,e),Do=!1},d(e){t(d),e&&t(g),e&&t(c),$(Q),e&&t(zt),e&&t(Te),e&&t(Bt),e&&t(k),e&&t(Wt),e&&t(L),e&&t(Ht),e&&t(R),e&&t(Yt),e&&t(qe),e&&t(Vt),e&&t(S),e&&t(Kt),$(N,e),e&&t(Xt),e&&t(j),$(le),e&&t(Zt),e&&t(Ie),e&&t(Jt),e&&t(D),e&&t(Ft),e&&t(Ce),e&&t(Qt),e&&t(M),e&&t(eo),e&&t(Ne),e&&t(to),$(he,e),e&&t(oo),e&&t(U),e&&t(ao),e&&t(Oe),e&&t(io),$(G,e),e&&t(no),e&&t(Me),e&&t(lo),e&&t(z),e&&t(ro),e&&t(B),e&&t(so),e&&t(W),e&&t(fo),e&&t(T),$(_e),e&&t(po),e&&t(P),e&&t(uo),$(ve,e),e&&t(ho),e&&t(ze),e&&t(co),e&&t(Be),e&&t(mo),e&&t(q),e&&t(go),e&&t(x),e&&t(_o),$($e,e),e&&t(wo),e&&t(Y),e&&t(vo),e&&t(V),e&&t(yo),e&&t(K),e&&t(bo),e&&t(I),$(Le),e&&t(Eo),e&&t(We),e&&t(Ao),e&&t(_),e&&t($o),$(Re,e),e&&t(ko),e&&t(w),e&&t(Lo),e&&t(C),$(De),e&&t(Ro),e&&t(He)}}}const kl={local:"lora-support-in-diffusers",sections:[{local:"getting-started-with-lora-for-finetuning",title:"Getting started with LoRA for fine-tuning"},{local:"inference",sections:[{local:"merging-lora-with-original-model",title:"Merging LoRA with original model"}],title:"Inference "},{local:"known-limitations",title:"Known limitations "}],title:"LoRA Support in Diffusers "};function Ll(je){return bl(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Sl extends gl{constructor(d){super();_l(this,d,Ll,$l,wl,{})}}export{Sl as default,kl as metadata};
