import{S as ma,i as pa,s as ua,e as r,k as p,w,t as s,M as ha,c as a,d as t,m as u,a as i,x as y,h as n,b as f,N as jt,G as o,g as m,y as E,L as ca,q as $,o as P,B as x,v as da}from"../chunks/vendor-hf-doc-builder.js";import{I as le}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as ir}from"../chunks/CodeBlock-hf-doc-builder.js";function va(lr){let G,lt,A,T,qe,H,Ft,Me,Ht,st,d,Wt,se,Xt,zt,ne,Vt,Kt,nt,k,O,Ie,W,Zt,fe,Te,Jt,Qt,ft,v,Yt,Oe,eo,to,X,oo,ro,mt,U,B,Be,z,ao,me,Se,io,lo,pt,pe,so,ut,h,no,V,Ne,fo,mo,Le,po,uo,Re,ho,co,De,vo,bo,je,_o,go,ht,ue,he,sr,ct,b,wo,K,yo,Eo,Z,$o,Po,dt,ce,de,nr,vt,S,xo,ve,Go,Ao,bt,C,N,Fe,J,ko,He,Uo,_t,_,We,Co,qo,L,Mo,Xe,Io,To,ze,Oo,Bo,R,So,Ve,No,Lo,Ke,Ro,gt,q,D,Ze,Q,Do,Je,jo,wt,be,Fo,yt,Y,Et,M,j,Qe,ee,Ho,Ye,Wo,$t,_e,Xo,Pt,te,xt,g,zo,et,Vo,Ko,tt,Zo,Jo,Gt,oe,At,ge,Qo,kt,I,F,ot,re,Yo,rt,er,Ut,we,tr,Ct,ye,ae,Ee,fr,qt,$e,or,Mt,Pe,ie,xe,mr,It;return H=new le({}),W=new le({}),z=new le({}),J=new le({}),Q=new le({}),Y=new ir({props:{code:`model_name = "bigscience/bloom-2b5"
model_8bit = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto", load_in_8bit=True)`,highlighted:`model_name = <span class="hljs-string">&quot;bigscience/bloom-2b5&quot;</span>
model_8bit = AutoModelForCausalLM.from_pretrained(model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>)`}}),ee=new le({}),te=new ir({props:{code:`model_name = "bigscience/bloom-2b5"
model_8bit = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto", load_in_8bit=True)`,highlighted:`model_name = <span class="hljs-string">&quot;bigscience/bloom-2b5&quot;</span>
model_8bit = AutoModelForCausalLM.from_pretrained(model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>)`}}),oe=new ir({props:{code:`max_memory_mapping = {0: "1GB", 1: "2GB"}
model_name = "bigscience/bloom-3b"
model_8bit = AutoModelForCausalLM.from_pretrained(
    model_name, device_map="auto", load_in_8bit=True, max_memory=max_memory_mapping
)`,highlighted:`max_memory_mapping = {<span class="hljs-number">0</span>: <span class="hljs-string">&quot;1GB&quot;</span>, <span class="hljs-number">1</span>: <span class="hljs-string">&quot;2GB&quot;</span>}
model_name = <span class="hljs-string">&quot;bigscience/bloom-3b&quot;</span>
model_8bit = AutoModelForCausalLM.from_pretrained(
    model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>, max_memory=max_memory_mapping
)`}}),re=new le({}),{c(){G=r("meta"),lt=p(),A=r("h1"),T=r("a"),qe=r("span"),w(H.$$.fragment),Ft=p(),Me=r("span"),Ht=s("Efficient Inference on a Single GPU"),st=p(),d=r("p"),Wt=s("This document will be completed soon with information on how to infer on a single GPU. In the meantime you can check out "),se=r("a"),Xt=s("the guide for training on a single GPU"),zt=s(" and "),ne=r("a"),Vt=s("the guide for inference on CPUs"),Kt=s("."),nt=p(),k=r("h2"),O=r("a"),Ie=r("span"),w(W.$$.fragment),Zt=p(),fe=r("span"),Te=r("code"),Jt=s("BetterTransformer"),Qt=s(" for faster inference"),ft=p(),v=r("p"),Yt=s("We have recently integrated "),Oe=r("code"),eo=s("BetterTransformer"),to=s(" for faster inference on GPU for text, image and audio models. Check the documentation about this integration "),X=r("a"),oo=s("here"),ro=s(" for more details."),mt=p(),U=r("h2"),B=r("a"),Be=r("span"),w(z.$$.fragment),ao=p(),me=r("span"),Se=r("code"),io=s("bitsandbytes"),lo=s(" integration for Int8 mixed-precision matrix decomposition"),pt=p(),pe=r("p"),so=s("Note that this feature is also totally applicable in a multi GPU setup as well."),ut=p(),h=r("p"),no=s("From the paper "),V=r("a"),Ne=r("code"),fo=s("LLM.int8() : 8-bit Matrix Multiplication for Transformers at Scale"),mo=s(`, we support HuggingFace integration for all models in the Hub with a few lines of code.
The method reduce `),Le=r("code"),po=s("nn.Linear"),uo=s(" size by 2 for "),Re=r("code"),ho=s("float16"),co=s(" and "),De=r("code"),vo=s("bfloat16"),bo=s(" weights and by 4 for "),je=r("code"),_o=s("float32"),go=s(" weights, with close to no impact to the quality by operating on the outliers in half-precision."),ht=p(),ue=r("p"),he=r("img"),ct=p(),b=r("p"),wo=s(`Int8 mixed-precision matrix decomposition works by separating a matrix multiplication into two streams: (1) a systematic feature outlier stream matrix multiplied in fp16 (0.01%), (2) a regular stream of int8 matrix multiplication (99.9%). With this method, int8 inference with no predictive degradation is possible for very large models.
For more details regarding the method, check out the `),K=r("a"),yo=s("paper"),Eo=s(" or our "),Z=r("a"),$o=s("blogpost about the integration"),Po=s("."),dt=p(),ce=r("p"),de=r("img"),vt=p(),S=r("p"),xo=s(`Note, that you would require a GPU to run mixed-8bit models as the kernels have been compiled for GPUs only. Make sure that you have enough GPU memory to store the quarter (or half if your model weights are in half precision) of the model before using this feature.
Below are some notes to help you use this module, or follow the demos on `),ve=r("a"),Go=s("Google colab"),Ao=s("."),bt=p(),C=r("h3"),N=r("a"),Fe=r("span"),w(J.$$.fragment),ko=p(),He=r("span"),Uo=s("Requirements"),_t=p(),_=r("ul"),We=r("li"),Co=s("Make sure you run that on NVIDIA GPUs that support 8-bit tensor cores (Turing, Ampere or newer architectures - e.g. T4, RTX20s RTX30s, A40-A100)."),qo=p(),L=r("li"),Mo=s("Install the correct version of "),Xe=r("code"),Io=s("bitsandbytes"),To=s(` by running:
`),ze=r("code"),Oo=s("pip install bitsandbytes>=0.31.5"),Bo=p(),R=r("li"),So=s("Install "),Ve=r("code"),No=s("accelerate"),Lo=p(),Ke=r("code"),Ro=s("pip install accelerate>=0.12.0"),gt=p(),q=r("h3"),D=r("a"),Ze=r("span"),w(Q.$$.fragment),Do=p(),Je=r("span"),jo=s("Running mixed-int8 models - single GPU setup"),wt=p(),be=r("p"),Fo=s("After installing the required libraries, the way to load your mixed 8-bit model is as follows:"),yt=p(),w(Y.$$.fragment),Et=p(),M=r("h3"),j=r("a"),Qe=r("span"),w(ee.$$.fragment),Ho=p(),Ye=r("span"),Wo=s("Running mixed-int8 models - multi GPU setup"),$t=p(),_e=r("p"),Xo=s("The way to load your mixed 8-bit model in multiple GPUs is as follows (same command as single GPU setup):"),Pt=p(),w(te.$$.fragment),xt=p(),g=r("p"),zo=s("But you can control the GPU RAM you want to allocate on each GPU using "),et=r("code"),Vo=s("accelerate"),Ko=s(". Use the "),tt=r("code"),Zo=s("max_memory"),Jo=s(" argument as follows:"),Gt=p(),w(oe.$$.fragment),At=p(),ge=r("p"),Qo=s("In this example, the first GPU will use 1GB of memory and the second 2GB."),kt=p(),I=r("h3"),F=r("a"),ot=r("span"),w(re.$$.fragment),Yo=p(),rt=r("span"),er=s("Colab demos"),Ut=p(),we=r("p"),tr=s(`With this method you can infer on models that were not possible to infer on a Google Colab before.
Check out the demo for running T5-11b (42GB in fp32)! Using 8-bit quantization on Google Colab:`),Ct=p(),ye=r("p"),ae=r("a"),Ee=r("img"),qt=p(),$e=r("p"),or=s("Or this demo for BLOOM-3B:"),Mt=p(),Pe=r("p"),ie=r("a"),xe=r("img"),this.h()},l(e){const l=ha('[data-svelte="svelte-1phssyn"]',document.head);G=a(l,"META",{name:!0,content:!0}),l.forEach(t),lt=u(e),A=a(e,"H1",{class:!0});var Tt=i(A);T=a(Tt,"A",{id:!0,class:!0,href:!0});var pr=i(T);qe=a(pr,"SPAN",{});var ur=i(qe);y(H.$$.fragment,ur),ur.forEach(t),pr.forEach(t),Ft=u(Tt),Me=a(Tt,"SPAN",{});var hr=i(Me);Ht=n(hr,"Efficient Inference on a Single GPU"),hr.forEach(t),Tt.forEach(t),st=u(e),d=a(e,"P",{});var Ge=i(d);Wt=n(Ge,"This document will be completed soon with information on how to infer on a single GPU. In the meantime you can check out "),se=a(Ge,"A",{href:!0});var cr=i(se);Xt=n(cr,"the guide for training on a single GPU"),cr.forEach(t),zt=n(Ge," and "),ne=a(Ge,"A",{href:!0});var dr=i(ne);Vt=n(dr,"the guide for inference on CPUs"),dr.forEach(t),Kt=n(Ge,"."),Ge.forEach(t),nt=u(e),k=a(e,"H2",{class:!0});var Ot=i(k);O=a(Ot,"A",{id:!0,class:!0,href:!0});var vr=i(O);Ie=a(vr,"SPAN",{});var br=i(Ie);y(W.$$.fragment,br),br.forEach(t),vr.forEach(t),Zt=u(Ot),fe=a(Ot,"SPAN",{});var rr=i(fe);Te=a(rr,"CODE",{});var _r=i(Te);Jt=n(_r,"BetterTransformer"),_r.forEach(t),Qt=n(rr," for faster inference"),rr.forEach(t),Ot.forEach(t),ft=u(e),v=a(e,"P",{});var Ae=i(v);Yt=n(Ae,"We have recently integrated "),Oe=a(Ae,"CODE",{});var gr=i(Oe);eo=n(gr,"BetterTransformer"),gr.forEach(t),to=n(Ae," for faster inference on GPU for text, image and audio models. Check the documentation about this integration "),X=a(Ae,"A",{href:!0,rel:!0});var wr=i(X);oo=n(wr,"here"),wr.forEach(t),ro=n(Ae," for more details."),Ae.forEach(t),mt=u(e),U=a(e,"H2",{class:!0});var Bt=i(U);B=a(Bt,"A",{id:!0,class:!0,href:!0});var yr=i(B);Be=a(yr,"SPAN",{});var Er=i(Be);y(z.$$.fragment,Er),Er.forEach(t),yr.forEach(t),ao=u(Bt),me=a(Bt,"SPAN",{});var ar=i(me);Se=a(ar,"CODE",{});var $r=i(Se);io=n($r,"bitsandbytes"),$r.forEach(t),lo=n(ar," integration for Int8 mixed-precision matrix decomposition"),ar.forEach(t),Bt.forEach(t),pt=u(e),pe=a(e,"P",{});var Pr=i(pe);so=n(Pr,"Note that this feature is also totally applicable in a multi GPU setup as well."),Pr.forEach(t),ut=u(e),h=a(e,"P",{});var c=i(h);no=n(c,"From the paper "),V=a(c,"A",{href:!0,rel:!0});var xr=i(V);Ne=a(xr,"CODE",{});var Gr=i(Ne);fo=n(Gr,"LLM.int8() : 8-bit Matrix Multiplication for Transformers at Scale"),Gr.forEach(t),xr.forEach(t),mo=n(c,`, we support HuggingFace integration for all models in the Hub with a few lines of code.
The method reduce `),Le=a(c,"CODE",{});var Ar=i(Le);po=n(Ar,"nn.Linear"),Ar.forEach(t),uo=n(c," size by 2 for "),Re=a(c,"CODE",{});var kr=i(Re);ho=n(kr,"float16"),kr.forEach(t),co=n(c," and "),De=a(c,"CODE",{});var Ur=i(De);vo=n(Ur,"bfloat16"),Ur.forEach(t),bo=n(c," weights and by 4 for "),je=a(c,"CODE",{});var Cr=i(je);_o=n(Cr,"float32"),Cr.forEach(t),go=n(c," weights, with close to no impact to the quality by operating on the outliers in half-precision."),c.forEach(t),ht=u(e),ue=a(e,"P",{});var qr=i(ue);he=a(qr,"IMG",{src:!0,alt:!0}),qr.forEach(t),ct=u(e),b=a(e,"P",{});var ke=i(b);wo=n(ke,`Int8 mixed-precision matrix decomposition works by separating a matrix multiplication into two streams: (1) a systematic feature outlier stream matrix multiplied in fp16 (0.01%), (2) a regular stream of int8 matrix multiplication (99.9%). With this method, int8 inference with no predictive degradation is possible for very large models.
For more details regarding the method, check out the `),K=a(ke,"A",{href:!0,rel:!0});var Mr=i(K);yo=n(Mr,"paper"),Mr.forEach(t),Eo=n(ke," or our "),Z=a(ke,"A",{href:!0,rel:!0});var Ir=i(Z);$o=n(Ir,"blogpost about the integration"),Ir.forEach(t),Po=n(ke,"."),ke.forEach(t),dt=u(e),ce=a(e,"P",{});var Tr=i(ce);de=a(Tr,"IMG",{src:!0,alt:!0}),Tr.forEach(t),vt=u(e),S=a(e,"P",{});var St=i(S);xo=n(St,`Note, that you would require a GPU to run mixed-8bit models as the kernels have been compiled for GPUs only. Make sure that you have enough GPU memory to store the quarter (or half if your model weights are in half precision) of the model before using this feature.
Below are some notes to help you use this module, or follow the demos on `),ve=a(St,"A",{href:!0});var Or=i(ve);Go=n(Or,"Google colab"),Or.forEach(t),Ao=n(St,"."),St.forEach(t),bt=u(e),C=a(e,"H3",{class:!0});var Nt=i(C);N=a(Nt,"A",{id:!0,class:!0,href:!0});var Br=i(N);Fe=a(Br,"SPAN",{});var Sr=i(Fe);y(J.$$.fragment,Sr),Sr.forEach(t),Br.forEach(t),ko=u(Nt),He=a(Nt,"SPAN",{});var Nr=i(He);Uo=n(Nr,"Requirements"),Nr.forEach(t),Nt.forEach(t),_t=u(e),_=a(e,"UL",{});var Ue=i(_);We=a(Ue,"LI",{});var Lr=i(We);Co=n(Lr,"Make sure you run that on NVIDIA GPUs that support 8-bit tensor cores (Turing, Ampere or newer architectures - e.g. T4, RTX20s RTX30s, A40-A100)."),Lr.forEach(t),qo=u(Ue),L=a(Ue,"LI",{});var at=i(L);Mo=n(at,"Install the correct version of "),Xe=a(at,"CODE",{});var Rr=i(Xe);Io=n(Rr,"bitsandbytes"),Rr.forEach(t),To=n(at,` by running:
`),ze=a(at,"CODE",{});var Dr=i(ze);Oo=n(Dr,"pip install bitsandbytes>=0.31.5"),Dr.forEach(t),at.forEach(t),Bo=u(Ue),R=a(Ue,"LI",{});var it=i(R);So=n(it,"Install "),Ve=a(it,"CODE",{});var jr=i(Ve);No=n(jr,"accelerate"),jr.forEach(t),Lo=u(it),Ke=a(it,"CODE",{});var Fr=i(Ke);Ro=n(Fr,"pip install accelerate>=0.12.0"),Fr.forEach(t),it.forEach(t),Ue.forEach(t),gt=u(e),q=a(e,"H3",{class:!0});var Lt=i(q);D=a(Lt,"A",{id:!0,class:!0,href:!0});var Hr=i(D);Ze=a(Hr,"SPAN",{});var Wr=i(Ze);y(Q.$$.fragment,Wr),Wr.forEach(t),Hr.forEach(t),Do=u(Lt),Je=a(Lt,"SPAN",{});var Xr=i(Je);jo=n(Xr,"Running mixed-int8 models - single GPU setup"),Xr.forEach(t),Lt.forEach(t),wt=u(e),be=a(e,"P",{});var zr=i(be);Fo=n(zr,"After installing the required libraries, the way to load your mixed 8-bit model is as follows:"),zr.forEach(t),yt=u(e),y(Y.$$.fragment,e),Et=u(e),M=a(e,"H3",{class:!0});var Rt=i(M);j=a(Rt,"A",{id:!0,class:!0,href:!0});var Vr=i(j);Qe=a(Vr,"SPAN",{});var Kr=i(Qe);y(ee.$$.fragment,Kr),Kr.forEach(t),Vr.forEach(t),Ho=u(Rt),Ye=a(Rt,"SPAN",{});var Zr=i(Ye);Wo=n(Zr,"Running mixed-int8 models - multi GPU setup"),Zr.forEach(t),Rt.forEach(t),$t=u(e),_e=a(e,"P",{});var Jr=i(_e);Xo=n(Jr,"The way to load your mixed 8-bit model in multiple GPUs is as follows (same command as single GPU setup):"),Jr.forEach(t),Pt=u(e),y(te.$$.fragment,e),xt=u(e),g=a(e,"P",{});var Ce=i(g);zo=n(Ce,"But you can control the GPU RAM you want to allocate on each GPU using "),et=a(Ce,"CODE",{});var Qr=i(et);Vo=n(Qr,"accelerate"),Qr.forEach(t),Ko=n(Ce,". Use the "),tt=a(Ce,"CODE",{});var Yr=i(tt);Zo=n(Yr,"max_memory"),Yr.forEach(t),Jo=n(Ce," argument as follows:"),Ce.forEach(t),Gt=u(e),y(oe.$$.fragment,e),At=u(e),ge=a(e,"P",{});var ea=i(ge);Qo=n(ea,"In this example, the first GPU will use 1GB of memory and the second 2GB."),ea.forEach(t),kt=u(e),I=a(e,"H3",{class:!0});var Dt=i(I);F=a(Dt,"A",{id:!0,class:!0,href:!0});var ta=i(F);ot=a(ta,"SPAN",{});var oa=i(ot);y(re.$$.fragment,oa),oa.forEach(t),ta.forEach(t),Yo=u(Dt),rt=a(Dt,"SPAN",{});var ra=i(rt);er=n(ra,"Colab demos"),ra.forEach(t),Dt.forEach(t),Ut=u(e),we=a(e,"P",{});var aa=i(we);tr=n(aa,`With this method you can infer on models that were not possible to infer on a Google Colab before.
Check out the demo for running T5-11b (42GB in fp32)! Using 8-bit quantization on Google Colab:`),aa.forEach(t),Ct=u(e),ye=a(e,"P",{});var ia=i(ye);ae=a(ia,"A",{href:!0,rel:!0});var la=i(ae);Ee=a(la,"IMG",{src:!0,alt:!0}),la.forEach(t),ia.forEach(t),qt=u(e),$e=a(e,"P",{});var sa=i($e);or=n(sa,"Or this demo for BLOOM-3B:"),sa.forEach(t),Mt=u(e),Pe=a(e,"P",{});var na=i(Pe);ie=a(na,"A",{href:!0,rel:!0});var fa=i(ie);xe=a(fa,"IMG",{src:!0,alt:!0}),fa.forEach(t),na.forEach(t),this.h()},h(){f(G,"name","hf:doc:metadata"),f(G,"content",JSON.stringify(ba)),f(T,"id","efficient-inference-on-a-single-gpu"),f(T,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(T,"href","#efficient-inference-on-a-single-gpu"),f(A,"class","relative group"),f(se,"href","perf_train_gpu_one"),f(ne,"href","perf_infer_cpu"),f(O,"id","bettertransformer-for-faster-inference"),f(O,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(O,"href","#bettertransformer-for-faster-inference"),f(k,"class","relative group"),f(X,"href","https://huggingface.co/docs/optimum/bettertransformer/overview"),f(X,"rel","nofollow"),f(B,"id","bitsandbytes-integration-for-int8-mixedprecision-matrix-decomposition"),f(B,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(B,"href","#bitsandbytes-integration-for-int8-mixedprecision-matrix-decomposition"),f(U,"class","relative group"),f(V,"href","https://arxiv.org/abs/2208.07339"),f(V,"rel","nofollow"),jt(he.src,sr="https://s3.amazonaws.com/moonup/production/uploads/1659861207959-62441d1d9fdefb55a0b7d12c.png")||f(he,"src",sr),f(he,"alt","HFxbitsandbytes.png"),f(K,"href","https://arxiv.org/abs/2208.07339"),f(K,"rel","nofollow"),f(Z,"href","https://huggingface.co/blog/hf-bitsandbytes-integration"),f(Z,"rel","nofollow"),jt(de.src,nr="https://s3.amazonaws.com/moonup/production/uploads/1660567469965-62441d1d9fdefb55a0b7d12c.gif")||f(de,"src",nr),f(de,"alt","MixedInt8.gif"),f(ve,"href","#colab-demos"),f(N,"id","requirements"),f(N,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(N,"href","#requirements"),f(C,"class","relative group"),f(D,"id","running-mixedint8-models-single-gpu-setup"),f(D,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(D,"href","#running-mixedint8-models-single-gpu-setup"),f(q,"class","relative group"),f(j,"id","running-mixedint8-models-multi-gpu-setup"),f(j,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(j,"href","#running-mixedint8-models-multi-gpu-setup"),f(M,"class","relative group"),f(F,"id","colab-demos"),f(F,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(F,"href","#colab-demos"),f(I,"class","relative group"),jt(Ee.src,fr="https://colab.research.google.com/assets/colab-badge.svg")||f(Ee,"src",fr),f(Ee,"alt","Open In Colab: T5-11b demo"),f(ae,"href","https://colab.research.google.com/drive/1YORPWx4okIHXnjW7MSAidXN29mPVNT7F?usp=sharing"),f(ae,"rel","nofollow"),jt(xe.src,mr="https://colab.research.google.com/assets/colab-badge.svg")||f(xe,"src",mr),f(xe,"alt","Open In Colab: BLOOM-3b demo"),f(ie,"href","https://colab.research.google.com/drive/1qOjXfQIAULfKvZqwCen8-MoWKGdSatZ4?usp=sharing"),f(ie,"rel","nofollow")},m(e,l){o(document.head,G),m(e,lt,l),m(e,A,l),o(A,T),o(T,qe),E(H,qe,null),o(A,Ft),o(A,Me),o(Me,Ht),m(e,st,l),m(e,d,l),o(d,Wt),o(d,se),o(se,Xt),o(d,zt),o(d,ne),o(ne,Vt),o(d,Kt),m(e,nt,l),m(e,k,l),o(k,O),o(O,Ie),E(W,Ie,null),o(k,Zt),o(k,fe),o(fe,Te),o(Te,Jt),o(fe,Qt),m(e,ft,l),m(e,v,l),o(v,Yt),o(v,Oe),o(Oe,eo),o(v,to),o(v,X),o(X,oo),o(v,ro),m(e,mt,l),m(e,U,l),o(U,B),o(B,Be),E(z,Be,null),o(U,ao),o(U,me),o(me,Se),o(Se,io),o(me,lo),m(e,pt,l),m(e,pe,l),o(pe,so),m(e,ut,l),m(e,h,l),o(h,no),o(h,V),o(V,Ne),o(Ne,fo),o(h,mo),o(h,Le),o(Le,po),o(h,uo),o(h,Re),o(Re,ho),o(h,co),o(h,De),o(De,vo),o(h,bo),o(h,je),o(je,_o),o(h,go),m(e,ht,l),m(e,ue,l),o(ue,he),m(e,ct,l),m(e,b,l),o(b,wo),o(b,K),o(K,yo),o(b,Eo),o(b,Z),o(Z,$o),o(b,Po),m(e,dt,l),m(e,ce,l),o(ce,de),m(e,vt,l),m(e,S,l),o(S,xo),o(S,ve),o(ve,Go),o(S,Ao),m(e,bt,l),m(e,C,l),o(C,N),o(N,Fe),E(J,Fe,null),o(C,ko),o(C,He),o(He,Uo),m(e,_t,l),m(e,_,l),o(_,We),o(We,Co),o(_,qo),o(_,L),o(L,Mo),o(L,Xe),o(Xe,Io),o(L,To),o(L,ze),o(ze,Oo),o(_,Bo),o(_,R),o(R,So),o(R,Ve),o(Ve,No),o(R,Lo),o(R,Ke),o(Ke,Ro),m(e,gt,l),m(e,q,l),o(q,D),o(D,Ze),E(Q,Ze,null),o(q,Do),o(q,Je),o(Je,jo),m(e,wt,l),m(e,be,l),o(be,Fo),m(e,yt,l),E(Y,e,l),m(e,Et,l),m(e,M,l),o(M,j),o(j,Qe),E(ee,Qe,null),o(M,Ho),o(M,Ye),o(Ye,Wo),m(e,$t,l),m(e,_e,l),o(_e,Xo),m(e,Pt,l),E(te,e,l),m(e,xt,l),m(e,g,l),o(g,zo),o(g,et),o(et,Vo),o(g,Ko),o(g,tt),o(tt,Zo),o(g,Jo),m(e,Gt,l),E(oe,e,l),m(e,At,l),m(e,ge,l),o(ge,Qo),m(e,kt,l),m(e,I,l),o(I,F),o(F,ot),E(re,ot,null),o(I,Yo),o(I,rt),o(rt,er),m(e,Ut,l),m(e,we,l),o(we,tr),m(e,Ct,l),m(e,ye,l),o(ye,ae),o(ae,Ee),m(e,qt,l),m(e,$e,l),o($e,or),m(e,Mt,l),m(e,Pe,l),o(Pe,ie),o(ie,xe),It=!0},p:ca,i(e){It||($(H.$$.fragment,e),$(W.$$.fragment,e),$(z.$$.fragment,e),$(J.$$.fragment,e),$(Q.$$.fragment,e),$(Y.$$.fragment,e),$(ee.$$.fragment,e),$(te.$$.fragment,e),$(oe.$$.fragment,e),$(re.$$.fragment,e),It=!0)},o(e){P(H.$$.fragment,e),P(W.$$.fragment,e),P(z.$$.fragment,e),P(J.$$.fragment,e),P(Q.$$.fragment,e),P(Y.$$.fragment,e),P(ee.$$.fragment,e),P(te.$$.fragment,e),P(oe.$$.fragment,e),P(re.$$.fragment,e),It=!1},d(e){t(G),e&&t(lt),e&&t(A),x(H),e&&t(st),e&&t(d),e&&t(nt),e&&t(k),x(W),e&&t(ft),e&&t(v),e&&t(mt),e&&t(U),x(z),e&&t(pt),e&&t(pe),e&&t(ut),e&&t(h),e&&t(ht),e&&t(ue),e&&t(ct),e&&t(b),e&&t(dt),e&&t(ce),e&&t(vt),e&&t(S),e&&t(bt),e&&t(C),x(J),e&&t(_t),e&&t(_),e&&t(gt),e&&t(q),x(Q),e&&t(wt),e&&t(be),e&&t(yt),x(Y,e),e&&t(Et),e&&t(M),x(ee),e&&t($t),e&&t(_e),e&&t(Pt),x(te,e),e&&t(xt),e&&t(g),e&&t(Gt),x(oe,e),e&&t(At),e&&t(ge),e&&t(kt),e&&t(I),x(re),e&&t(Ut),e&&t(we),e&&t(Ct),e&&t(ye),e&&t(qt),e&&t($e),e&&t(Mt),e&&t(Pe)}}}const ba={local:"efficient-inference-on-a-single-gpu",sections:[{local:"bettertransformer-for-faster-inference",title:"`BetterTransformer` for faster inference"},{local:"bitsandbytes-integration-for-int8-mixedprecision-matrix-decomposition",sections:[{local:"requirements",title:"Requirements"},{local:"running-mixedint8-models-single-gpu-setup",title:"Running mixed-int8 models - single GPU setup"},{local:"running-mixedint8-models-multi-gpu-setup",title:"Running mixed-int8 models - multi GPU setup"},{local:"colab-demos",title:"Colab demos"}],title:"`bitsandbytes` integration for Int8 mixed-precision matrix decomposition"}],title:"Efficient Inference on a Single GPU"};function _a(lr){return da(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ea extends ma{constructor(G){super();pa(this,G,_a,va,ua,{})}}export{Ea as default,ba as metadata};
