import{S as Nu,i as zu,s as Uu,e as n,k as u,w as q,t,M as Gu,c as r,d as a,m as c,x as E,a as l,h as o,b as k,N as Hu,f as Qo,G as s,g as d,y as x,o as _,p as Xo,q as g,B as y,v as Bu,n as Ko}from"../../chunks/vendor-hf-doc-builder.js";import{T as Lu}from"../../chunks/Tip-hf-doc-builder.js";import{Y as Yu}from"../../chunks/Youtube-hf-doc-builder.js";import{I as Pt}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as O}from"../../chunks/CodeBlock-hf-doc-builder.js";import{C as Fu}from"../../chunks/CourseFloatingBanner-hf-doc-builder.js";import{F as Vu}from"../../chunks/FrameworkSwitchCourse-hf-doc-builder.js";function Wu(U){let m,b;return m=new Fu({props:{chapter:5,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter5/section6_tf.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter5/section6_tf.ipynb"}]}}),{c(){q(m.$$.fragment)},l(f){E(m.$$.fragment,f)},m(f,j){x(m,f,j),b=!0},i(f){b||(g(m.$$.fragment,f),b=!0)},o(f){_(m.$$.fragment,f),b=!1},d(f){y(m,f)}}}function Ju(U){let m,b;return m=new Fu({props:{chapter:5,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter5/section6_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter5/section6_pt.ipynb"}]}}),{c(){q(m.$$.fragment)},l(f){E(m.$$.fragment,f)},m(f,j){x(m,f,j),b=!0},i(f){b||(g(m.$$.fragment,f),b=!0)},o(f){_(m.$$.fragment,f),b=!1},d(f){y(m,f)}}}function Qu(U){let m,b,f,j,h,v,R,C,I,w,$,D,P,M,H,F,B,A,L,W;return{c(){m=n("p"),b=t("\u270F\uFE0F "),f=n("strong"),j=t("\xA1Int\xE9ntalo!"),h=t(" Prueba si puedes usar la funci\xF3n "),v=n("code"),R=t("Dataset.map()"),C=t(" para \u201Cexplotar\u201D la columna "),I=n("code"),w=t("comments"),$=t(" en "),D=n("code"),P=t("issues_dataset"),M=u(),H=n("em"),F=t("sin"),B=t(" necesidad de usar Pandas. Esto es un poco complejo; te recomendamos revisar la secci\xF3n de "),A=n("a"),L=t("\u201CBatch mapping\u201D"),W=t(" de la documentaci\xF3n de \u{1F917} Datasets para completar esta tarea."),this.h()},l(G){m=r(G,"P",{});var T=l(m);b=o(T,"\u270F\uFE0F "),f=r(T,"STRONG",{});var Y=l(f);j=o(Y,"\xA1Int\xE9ntalo!"),Y.forEach(a),h=o(T," Prueba si puedes usar la funci\xF3n "),v=r(T,"CODE",{});var p=l(v);R=o(p,"Dataset.map()"),p.forEach(a),C=o(T," para \u201Cexplotar\u201D la columna "),I=r(T,"CODE",{});var S=l(I);w=o(S,"comments"),S.forEach(a),$=o(T," en "),D=r(T,"CODE",{});var N=l(D);P=o(N,"issues_dataset"),N.forEach(a),M=c(T),H=r(T,"EM",{});var J=l(H);F=o(J,"sin"),J.forEach(a),B=o(T," necesidad de usar Pandas. Esto es un poco complejo; te recomendamos revisar la secci\xF3n de "),A=r(T,"A",{href:!0,rel:!0});var le=l(A);L=o(le,"\u201CBatch mapping\u201D"),le.forEach(a),W=o(T," de la documentaci\xF3n de \u{1F917} Datasets para completar esta tarea."),T.forEach(a),this.h()},h(){k(A,"href","https://huggingface.co/docs/datasets/v1.12.1/about_map_batch.html?batch-mapping#batch-mapping"),k(A,"rel","nofollow")},m(G,T){d(G,m,T),s(m,b),s(m,f),s(f,j),s(m,h),s(m,v),s(v,R),s(m,C),s(m,I),s(I,w),s(m,$),s(m,D),s(D,P),s(m,M),s(m,H),s(H,F),s(m,B),s(m,A),s(A,L),s(m,W)},d(G){G&&a(m)}}}function Xu(U){let m,b,f,j,h,v,R,C,I,w,$,D,P,M,H,F,B;return m=new O({props:{code:`from transformers import AutoTokenizer, TFAutoModel

model_ckpt = "sentence-transformers/multi-qa-mpnet-base-dot-v1"
tokenizer = AutoTokenizer.from_pretrained(model_ckpt)
model = TFAutoModel.from_pretrained(model_ckpt, from_pt=True)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, TFAutoModel

model_ckpt = <span class="hljs-string">&quot;sentence-transformers/multi-qa-mpnet-base-dot-v1&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_ckpt)
model = TFAutoModel.from_pretrained(model_ckpt, from_pt=<span class="hljs-literal">True</span>)`}}),{c(){q(m.$$.fragment),b=u(),f=n("p"),j=t("Ten en cuenta que hemos definido "),h=n("code"),v=t("from_pt=True"),R=t(" como un argumento del m\xE9todo "),C=n("code"),I=t("from_pretrained()"),w=t(". Esto es porque el punto de control "),$=n("code"),D=t("multi-qa-mpnet-base-dot-v1"),P=t(" s\xF3lo tiene pesos de PyTorch, asi que usar "),M=n("code"),H=t("from_pt=True"),F=t(" los va a covertir autom\xE1ticamente al formato TensorFlow. Como puedes ver, \xA1es m\xFAy f\xE1cil cambiar entre frameworks usando \u{1F917} Transformers!")},l(A){E(m.$$.fragment,A),b=c(A),f=r(A,"P",{});var L=l(f);j=o(L,"Ten en cuenta que hemos definido "),h=r(L,"CODE",{});var W=l(h);v=o(W,"from_pt=True"),W.forEach(a),R=o(L," como un argumento del m\xE9todo "),C=r(L,"CODE",{});var G=l(C);I=o(G,"from_pretrained()"),G.forEach(a),w=o(L,". Esto es porque el punto de control "),$=r(L,"CODE",{});var T=l($);D=o(T,"multi-qa-mpnet-base-dot-v1"),T.forEach(a),P=o(L," s\xF3lo tiene pesos de PyTorch, asi que usar "),M=r(L,"CODE",{});var Y=l(M);H=o(Y,"from_pt=True"),Y.forEach(a),F=o(L," los va a covertir autom\xE1ticamente al formato TensorFlow. Como puedes ver, \xA1es m\xFAy f\xE1cil cambiar entre frameworks usando \u{1F917} Transformers!"),L.forEach(a)},m(A,L){x(m,A,L),d(A,b,L),d(A,f,L),s(f,j),s(f,h),s(h,v),s(f,R),s(f,C),s(C,I),s(f,w),s(f,$),s($,D),s(f,P),s(f,M),s(M,H),s(f,F),B=!0},i(A){B||(g(m.$$.fragment,A),B=!0)},o(A){_(m.$$.fragment,A),B=!1},d(A){y(m,A),A&&a(b),A&&a(f)}}}function Ku(U){let m,b,f,j,h,v,R,C,I,w;return m=new O({props:{code:`from transformers import AutoTokenizer, AutoModel

model_ckpt = "sentence-transformers/multi-qa-mpnet-base-dot-v1"
tokenizer = AutoTokenizer.from_pretrained(model_ckpt)
model = AutoModel.from_pretrained(model_ckpt)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModel

model_ckpt = <span class="hljs-string">&quot;sentence-transformers/multi-qa-mpnet-base-dot-v1&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_ckpt)
model = AutoModel.from_pretrained(model_ckpt)`}}),I=new O({props:{code:`import torch

device = torch.device("cuda")
model.to(device)`,highlighted:`<span class="hljs-keyword">import</span> torch

device = torch.device(<span class="hljs-string">&quot;cuda&quot;</span>)
model.to(device)`}}),{c(){q(m.$$.fragment),b=u(),f=n("p"),j=t("Para acelerar el proceso de "),h=n("em"),v=t("embedding"),R=t(", es \xFAtil ubicar el modelo y los inputs en un dispositivo GPU, as\xED que hag\xE1moslo:"),C=u(),q(I.$$.fragment)},l($){E(m.$$.fragment,$),b=c($),f=r($,"P",{});var D=l(f);j=o(D,"Para acelerar el proceso de "),h=r(D,"EM",{});var P=l(h);v=o(P,"embedding"),P.forEach(a),R=o(D,", es \xFAtil ubicar el modelo y los inputs en un dispositivo GPU, as\xED que hag\xE1moslo:"),D.forEach(a),C=c($),E(I.$$.fragment,$)},m($,D){x(m,$,D),d($,b,D),d($,f,D),s(f,j),s(f,h),s(h,v),s(f,R),d($,C,D),x(I,$,D),w=!0},i($){w||(g(m.$$.fragment,$),g(I.$$.fragment,$),w=!0)},o($){_(m.$$.fragment,$),_(I.$$.fragment,$),w=!1},d($){y(m,$),$&&a(b),$&&a(f),$&&a(C),y(I,$)}}}function Zu(U){let m,b,f,j,h,v,R,C,I,w,$,D,P,M,H,F,B,A,L,W,G,T,Y;return m=new O({props:{code:`def get_embeddings(text_list):
    encoded_input = tokenizer(
        text_list, padding=True, truncation=True, return_tensors="tf"
    )
    encoded_input = {k: v for k, v in encoded_input.items()}
    model_output = model(**encoded_input)
    return cls_pooling(model_output)`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_embeddings</span>(<span class="hljs-params">text_list</span>):
    encoded_input = tokenizer(
        text_list, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>
    )
    encoded_input = {k: v <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> encoded_input.items()}
    model_output = model(**encoded_input)
    <span class="hljs-keyword">return</span> cls_pooling(model_output)`}}),v=new O({props:{code:`embedding = get_embeddings(comments_dataset["text"][0])
embedding.shape`,highlighted:`embedding = get_embeddings(comments_dataset[<span class="hljs-string">&quot;text&quot;</span>][<span class="hljs-number">0</span>])
embedding.shape`}}),C=new O({props:{code:"TensorShape([1, 768])",highlighted:'TensorShape([<span class="hljs-number">1</span>, <span class="hljs-number">768</span>])'}}),T=new O({props:{code:`embeddings_dataset = comments_dataset.map(
    lambda x: {"embeddings": get_embeddings(x["text"]).numpy()[0]}
)`,highlighted:`embeddings_dataset = comments_dataset.<span class="hljs-built_in">map</span>(
    <span class="hljs-keyword">lambda</span> x: {<span class="hljs-string">&quot;embeddings&quot;</span>: get_embeddings(x[<span class="hljs-string">&quot;text&quot;</span>]).numpy()[<span class="hljs-number">0</span>]}
)`}}),{c(){q(m.$$.fragment),b=u(),f=n("p"),j=t("Podemos probar que la funci\xF3n sirve al pasarle la primera entrada de texto en el corpus e inspeccionando la forma de la salida:"),h=u(),q(v.$$.fragment),R=u(),q(C.$$.fragment),I=u(),w=n("p"),$=t("\xA1Hemos convertido la primera entrada del corpus en un vector de 768 dimensiones! Ahora podemos usar "),D=n("code"),P=t("Dataset.map()"),M=t(" para aplicar nuestra funci\xF3n "),H=n("code"),F=t("get_embeddings()"),B=t(" a cada fila del corpus, as\xED que creemos una columna "),A=n("code"),L=t("embeddings"),W=t(" as\xED:"),G=u(),q(T.$$.fragment)},l(p){E(m.$$.fragment,p),b=c(p),f=r(p,"P",{});var S=l(f);j=o(S,"Podemos probar que la funci\xF3n sirve al pasarle la primera entrada de texto en el corpus e inspeccionando la forma de la salida:"),S.forEach(a),h=c(p),E(v.$$.fragment,p),R=c(p),E(C.$$.fragment,p),I=c(p),w=r(p,"P",{});var N=l(w);$=o(N,"\xA1Hemos convertido la primera entrada del corpus en un vector de 768 dimensiones! Ahora podemos usar "),D=r(N,"CODE",{});var J=l(D);P=o(J,"Dataset.map()"),J.forEach(a),M=o(N," para aplicar nuestra funci\xF3n "),H=r(N,"CODE",{});var le=l(H);F=o(le,"get_embeddings()"),le.forEach(a),B=o(N," a cada fila del corpus, as\xED que creemos una columna "),A=r(N,"CODE",{});var he=l(A);L=o(he,"embeddings"),he.forEach(a),W=o(N," as\xED:"),N.forEach(a),G=c(p),E(T.$$.fragment,p)},m(p,S){x(m,p,S),d(p,b,S),d(p,f,S),s(f,j),d(p,h,S),x(v,p,S),d(p,R,S),x(C,p,S),d(p,I,S),d(p,w,S),s(w,$),s(w,D),s(D,P),s(w,M),s(w,H),s(H,F),s(w,B),s(w,A),s(A,L),s(w,W),d(p,G,S),x(T,p,S),Y=!0},i(p){Y||(g(m.$$.fragment,p),g(v.$$.fragment,p),g(C.$$.fragment,p),g(T.$$.fragment,p),Y=!0)},o(p){_(m.$$.fragment,p),_(v.$$.fragment,p),_(C.$$.fragment,p),_(T.$$.fragment,p),Y=!1},d(p){y(m,p),p&&a(b),p&&a(f),p&&a(h),y(v,p),p&&a(R),y(C,p),p&&a(I),p&&a(w),p&&a(G),y(T,p)}}}function ec(U){let m,b,f,j,h,v,R,C,I,w,$,D,P,M,H,F,B,A,L,W,G,T,Y;return m=new O({props:{code:`def get_embeddings(text_list):
    encoded_input = tokenizer(
        text_list, padding=True, truncation=True, return_tensors="pt"
    )
    encoded_input = {k: v.to(device) for k, v in encoded_input.items()}
    model_output = model(**encoded_input)
    return cls_pooling(model_output)`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_embeddings</span>(<span class="hljs-params">text_list</span>):
    encoded_input = tokenizer(
        text_list, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>
    )
    encoded_input = {k: v.to(device) <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> encoded_input.items()}
    model_output = model(**encoded_input)
    <span class="hljs-keyword">return</span> cls_pooling(model_output)`}}),v=new O({props:{code:`embedding = get_embeddings(comments_dataset["text"][0])
embedding.shape`,highlighted:`embedding = get_embeddings(comments_dataset[<span class="hljs-string">&quot;text&quot;</span>][<span class="hljs-number">0</span>])
embedding.shape`}}),C=new O({props:{code:"torch.Size([1, 768])",highlighted:'torch.Size([<span class="hljs-number">1</span>, <span class="hljs-number">768</span>])'}}),T=new O({props:{code:`embeddings_dataset = comments_dataset.map(
    lambda x: {"embeddings": get_embeddings(x["text"]).detach().cpu().numpy()[0]}
)`,highlighted:`embeddings_dataset = comments_dataset.<span class="hljs-built_in">map</span>(
    <span class="hljs-keyword">lambda</span> x: {<span class="hljs-string">&quot;embeddings&quot;</span>: get_embeddings(x[<span class="hljs-string">&quot;text&quot;</span>]).detach().cpu().numpy()[<span class="hljs-number">0</span>]}
)`}}),{c(){q(m.$$.fragment),b=u(),f=n("p"),j=t("Podemos probar que la funci\xF3n sirve al pasarle la primera entrada de texto en el corpus e inspeccionando la forma de la salida:"),h=u(),q(v.$$.fragment),R=u(),q(C.$$.fragment),I=u(),w=n("p"),$=t("\xA1Hemos convertido la primera entrada del corpus en un vector de 768 dimensiones! Ahora podemos usar "),D=n("code"),P=t("Dataset.map()"),M=t(" para aplicar nuestra funci\xF3n "),H=n("code"),F=t("get_embeddings()"),B=t(" a cada fila del corpus, as\xED que creemos una columna "),A=n("code"),L=t("embeddings"),W=t(" as\xED:"),G=u(),q(T.$$.fragment)},l(p){E(m.$$.fragment,p),b=c(p),f=r(p,"P",{});var S=l(f);j=o(S,"Podemos probar que la funci\xF3n sirve al pasarle la primera entrada de texto en el corpus e inspeccionando la forma de la salida:"),S.forEach(a),h=c(p),E(v.$$.fragment,p),R=c(p),E(C.$$.fragment,p),I=c(p),w=r(p,"P",{});var N=l(w);$=o(N,"\xA1Hemos convertido la primera entrada del corpus en un vector de 768 dimensiones! Ahora podemos usar "),D=r(N,"CODE",{});var J=l(D);P=o(J,"Dataset.map()"),J.forEach(a),M=o(N," para aplicar nuestra funci\xF3n "),H=r(N,"CODE",{});var le=l(H);F=o(le,"get_embeddings()"),le.forEach(a),B=o(N," a cada fila del corpus, as\xED que creemos una columna "),A=r(N,"CODE",{});var he=l(A);L=o(he,"embeddings"),he.forEach(a),W=o(N," as\xED:"),N.forEach(a),G=c(p),E(T.$$.fragment,p)},m(p,S){x(m,p,S),d(p,b,S),d(p,f,S),s(f,j),d(p,h,S),x(v,p,S),d(p,R,S),x(C,p,S),d(p,I,S),d(p,w,S),s(w,$),s(w,D),s(D,P),s(w,M),s(w,H),s(H,F),s(w,B),s(w,A),s(A,L),s(w,W),d(p,G,S),x(T,p,S),Y=!0},i(p){Y||(g(m.$$.fragment,p),g(v.$$.fragment,p),g(C.$$.fragment,p),g(T.$$.fragment,p),Y=!0)},o(p){_(m.$$.fragment,p),_(v.$$.fragment,p),_(C.$$.fragment,p),_(T.$$.fragment,p),Y=!1},d(p){y(m,p),p&&a(b),p&&a(f),p&&a(h),y(v,p),p&&a(R),y(C,p),p&&a(I),p&&a(w),p&&a(G),y(T,p)}}}function sc(U){let m,b,f,j;return m=new O({props:{code:`question = "How can I load a dataset offline?"
question_embedding = get_embeddings([question]).numpy()
question_embedding.shape`,highlighted:`question = <span class="hljs-string">&quot;How can I load a dataset offline?&quot;</span>
question_embedding = get_embeddings([question]).numpy()
question_embedding.shape`}}),f=new O({props:{code:"(1, 768)",highlighted:'(<span class="hljs-number">1</span>, <span class="hljs-number">768</span>)'}}),{c(){q(m.$$.fragment),b=u(),q(f.$$.fragment)},l(h){E(m.$$.fragment,h),b=c(h),E(f.$$.fragment,h)},m(h,v){x(m,h,v),d(h,b,v),x(f,h,v),j=!0},i(h){j||(g(m.$$.fragment,h),g(f.$$.fragment,h),j=!0)},o(h){_(m.$$.fragment,h),_(f.$$.fragment,h),j=!1},d(h){y(m,h),h&&a(b),y(f,h)}}}function ac(U){let m,b,f,j;return m=new O({props:{code:`question = "How can I load a dataset offline?"
question_embedding = get_embeddings([question]).cpu().detach().numpy()
question_embedding.shape`,highlighted:`question = <span class="hljs-string">&quot;How can I load a dataset offline?&quot;</span>
question_embedding = get_embeddings([question]).cpu().detach().numpy()
question_embedding.shape`}}),f=new O({props:{code:"torch.Size([1, 768])",highlighted:'torch.Size([<span class="hljs-number">1</span>, <span class="hljs-number">768</span>])'}}),{c(){q(m.$$.fragment),b=u(),q(f.$$.fragment)},l(h){E(m.$$.fragment,h),b=c(h),E(f.$$.fragment,h)},m(h,v){x(m,h,v),d(h,b,v),x(f,h,v),j=!0},i(h){j||(g(m.$$.fragment,h),g(f.$$.fragment,h),j=!0)},o(h){_(m.$$.fragment,h),_(f.$$.fragment,h),j=!1},d(h){y(m,h),h&&a(b),y(f,h)}}}function tc(U){let m,b,f,j,h,v,R,C,I,w,$;return{c(){m=n("p"),b=t("\u270F\uFE0F "),f=n("strong"),j=t("\xA1Int\xE9ntalo!"),h=t(" Crea tu propia pregunta y prueba si puedes encontrar una respuesta en los documentos devueltos. Puede que tengas que incrementar el par\xE1metro "),v=n("code"),R=t("k"),C=t(" en "),I=n("code"),w=t("Dataset.get_nearest_examples()"),$=t(" para aumentar la b\xFAsqueda.")},l(D){m=r(D,"P",{});var P=l(m);b=o(P,"\u270F\uFE0F "),f=r(P,"STRONG",{});var M=l(f);j=o(M,"\xA1Int\xE9ntalo!"),M.forEach(a),h=o(P," Crea tu propia pregunta y prueba si puedes encontrar una respuesta en los documentos devueltos. Puede que tengas que incrementar el par\xE1metro "),v=r(P,"CODE",{});var H=l(v);R=o(H,"k"),H.forEach(a),C=o(P," en "),I=r(P,"CODE",{});var F=l(I);w=o(F,"Dataset.get_nearest_examples()"),F.forEach(a),$=o(P," para aumentar la b\xFAsqueda."),P.forEach(a)},m(D,P){d(D,m,P),s(m,b),s(m,f),s(f,j),s(m,h),s(m,v),s(v,R),s(m,C),s(m,I),s(I,w),s(m,$)},d(D){D&&a(m)}}}function oc(U){let m,b,f,j,h,v,R,C,I,w,$,D,P,M,H,F,B,A,L,W,G,T,Y,p,S,N,J,le,he,Zo,St,Q,en,Hs,sn,an,ta,tn,on,oa,nn,rn,na,ln,dn,ra,un,cn,Ot,Re,mn,la,pn,fn,It,Se,ss,Mi,hn,as,Ri,Mt,Oe,He,ia,ts,_n,da,gn,Rt,Ls,bn,Ht,os,Lt,xe,vn,ua,$n,qn,Fs,En,xn,Ft,ns,Nt,rs,zt,X,yn,ca,wn,kn,ma,jn,Dn,pa,Tn,Cn,fa,An,Pn,ha,Sn,On,Ut,ls,Gt,is,Bt,K,In,_a,Mn,Rn,ga,Hn,Ln,ba,Fn,Nn,va,zn,Un,$a,Gn,Bn,Yt,ds,Vt,us,Wt,Z,Yn,qa,Vn,Wn,Ea,Jn,Qn,xa,Xn,Kn,Le,Zn,ya,er,sr,wa,ar,tr,Jt,cs,Qt,Fe,or,ka,nr,rr,Xt,ms,Kt,ps,Zt,Ne,lr,ja,ir,dr,eo,fs,so,ae,Da,ee,ao,ur,Ta,cr,mr,Ca,pr,fr,Aa,hr,_r,Pa,gr,br,_e,te,Sa,vr,$r,Oa,qr,Er,Ia,xr,yr,Ma,wr,kr,Ra,jr,Dr,oe,Ha,Tr,Cr,La,Ar,Pr,Fa,Sr,Or,Na,Ir,Mr,za,Rr,Hr,ne,Ua,Lr,Fr,Ga,Nr,zr,Ba,Ur,Gr,Ya,Br,Yr,Va,Vr,Wr,re,Wa,Jr,Qr,Ja,Xr,Kr,Qa,Zr,el,Xa,sl,al,Ka,tl,to,ie,ol,Za,nl,rl,et,ll,il,st,dl,ul,oo,hs,no,_s,ro,Ns,cl,lo,ze,io,Ue,ml,at,pl,fl,uo,gs,co,zs,hl,mo,bs,po,vs,fo,ye,_l,tt,gl,bl,ot,vl,$l,ho,$s,_o,Ge,ql,nt,El,xl,go,Ie,Be,rt,qs,yl,lt,wl,bo,z,kl,Us,jl,Dl,it,Tl,Cl,dt,Al,Pl,ut,Sl,Ol,ct,Il,Ml,Es,Rl,Hl,mt,Ll,Fl,xs,Nl,zl,pt,Ul,Gl,vo,ge,be,Gs,de,Bl,ft,Yl,Vl,ht,Wl,Jl,_t,Ql,Xl,$o,ys,qo,Bs,Kl,Eo,ve,$e,Ys,Ye,Zl,gt,ei,si,xo,Me,Ve,bt,ws,ai,vt,ti,yo,se,oi,$t,ni,ri,ks,li,ii,qt,di,ui,Et,ci,mi,wo,ue,pi,xt,fi,hi,yt,_i,gi,wt,bi,vi,ko,js,jo,we,$i,kt,qi,Ei,jt,xi,yi,Do,qe,Ee,Vs,We,wi,Dt,ki,ji,To,Ds,Co,ke,Di,Tt,Ti,Ci,Ct,Ai,Pi,Ao,Ts,Po,Ws,Si,So,Cs,Oo,As,Io,Js,Oi,Mo,Je,Ro;f=new Vu({props:{fw:U[0]}}),C=new Pt({});const Hi=[Ju,Wu],Ps=[];function Li(e,i){return e[0]==="pt"?0:1}P=Li(U),M=Ps[P]=Hi[P](U),T=new Yu({props:{id:"OATCgQtNX2o"}}),J=new Pt({}),ts=new Pt({}),os=new O({props:{code:`from huggingface_hub import hf_hub_url

data_files = hf_hub_url(
    repo_id="lewtun/github-issues",
    filename="datasets-issues-with-hf-doc-builder.jsonl",
    repo_type="dataset",
)`,highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> hf_hub_url

data_files = hf_hub_url(
    repo_id=<span class="hljs-string">&quot;lewtun/github-issues&quot;</span>,
    filename=<span class="hljs-string">&quot;datasets-issues-with-hf-doc-builder.jsonl&quot;</span>,
    repo_type=<span class="hljs-string">&quot;dataset&quot;</span>,
)`}}),ns=new O({props:{code:`from datasets import load_dataset

issues_dataset = load_dataset("json", data_files=data_files, split="train")
issues_dataset`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

issues_dataset = load_dataset(<span class="hljs-string">&quot;json&quot;</span>, data_files=data_files, split=<span class="hljs-string">&quot;train&quot;</span>)
issues_dataset`}}),rs=new O({props:{code:`Dataset({
    features: ['url', 'repository_url', 'labels_url', 'comments_url', 'events_url', 'html_url', 'id', 'node_id', 'number', 'title', 'user', 'labels', 'state', 'locked', 'assignee', 'assignees', 'milestone', 'comments', 'created_at', 'updated_at', 'closed_at', 'author_association', 'active_lock_reason', 'pull_request', 'body', 'performed_via_github_app', 'is_pull_request'],
    num_rows: 2855
})`,highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;url&#x27;</span>, <span class="hljs-string">&#x27;repository_url&#x27;</span>, <span class="hljs-string">&#x27;labels_url&#x27;</span>, <span class="hljs-string">&#x27;comments_url&#x27;</span>, <span class="hljs-string">&#x27;events_url&#x27;</span>, <span class="hljs-string">&#x27;html_url&#x27;</span>, <span class="hljs-string">&#x27;id&#x27;</span>, <span class="hljs-string">&#x27;node_id&#x27;</span>, <span class="hljs-string">&#x27;number&#x27;</span>, <span class="hljs-string">&#x27;title&#x27;</span>, <span class="hljs-string">&#x27;user&#x27;</span>, <span class="hljs-string">&#x27;labels&#x27;</span>, <span class="hljs-string">&#x27;state&#x27;</span>, <span class="hljs-string">&#x27;locked&#x27;</span>, <span class="hljs-string">&#x27;assignee&#x27;</span>, <span class="hljs-string">&#x27;assignees&#x27;</span>, <span class="hljs-string">&#x27;milestone&#x27;</span>, <span class="hljs-string">&#x27;comments&#x27;</span>, <span class="hljs-string">&#x27;created_at&#x27;</span>, <span class="hljs-string">&#x27;updated_at&#x27;</span>, <span class="hljs-string">&#x27;closed_at&#x27;</span>, <span class="hljs-string">&#x27;author_association&#x27;</span>, <span class="hljs-string">&#x27;active_lock_reason&#x27;</span>, <span class="hljs-string">&#x27;pull_request&#x27;</span>, <span class="hljs-string">&#x27;body&#x27;</span>, <span class="hljs-string">&#x27;performed_via_github_app&#x27;</span>, <span class="hljs-string">&#x27;is_pull_request&#x27;</span>],
    num_rows: <span class="hljs-number">2855</span>
})`}}),ls=new O({props:{code:`issues_dataset = issues_dataset.filter(
    lambda x: (x["is_pull_request"] == False and len(x["comments"]) > 0)
)
issues_dataset`,highlighted:`issues_dataset = issues_dataset.<span class="hljs-built_in">filter</span>(
    <span class="hljs-keyword">lambda</span> x: (x[<span class="hljs-string">&quot;is_pull_request&quot;</span>] == <span class="hljs-literal">False</span> <span class="hljs-keyword">and</span> <span class="hljs-built_in">len</span>(x[<span class="hljs-string">&quot;comments&quot;</span>]) &gt; <span class="hljs-number">0</span>)
)
issues_dataset`}}),is=new O({props:{code:`Dataset({
    features: ['url', 'repository_url', 'labels_url', 'comments_url', 'events_url', 'html_url', 'id', 'node_id', 'number', 'title', 'user', 'labels', 'state', 'locked', 'assignee', 'assignees', 'milestone', 'comments', 'created_at', 'updated_at', 'closed_at', 'author_association', 'active_lock_reason', 'pull_request', 'body', 'performed_via_github_app', 'is_pull_request'],
    num_rows: 771
})`,highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;url&#x27;</span>, <span class="hljs-string">&#x27;repository_url&#x27;</span>, <span class="hljs-string">&#x27;labels_url&#x27;</span>, <span class="hljs-string">&#x27;comments_url&#x27;</span>, <span class="hljs-string">&#x27;events_url&#x27;</span>, <span class="hljs-string">&#x27;html_url&#x27;</span>, <span class="hljs-string">&#x27;id&#x27;</span>, <span class="hljs-string">&#x27;node_id&#x27;</span>, <span class="hljs-string">&#x27;number&#x27;</span>, <span class="hljs-string">&#x27;title&#x27;</span>, <span class="hljs-string">&#x27;user&#x27;</span>, <span class="hljs-string">&#x27;labels&#x27;</span>, <span class="hljs-string">&#x27;state&#x27;</span>, <span class="hljs-string">&#x27;locked&#x27;</span>, <span class="hljs-string">&#x27;assignee&#x27;</span>, <span class="hljs-string">&#x27;assignees&#x27;</span>, <span class="hljs-string">&#x27;milestone&#x27;</span>, <span class="hljs-string">&#x27;comments&#x27;</span>, <span class="hljs-string">&#x27;created_at&#x27;</span>, <span class="hljs-string">&#x27;updated_at&#x27;</span>, <span class="hljs-string">&#x27;closed_at&#x27;</span>, <span class="hljs-string">&#x27;author_association&#x27;</span>, <span class="hljs-string">&#x27;active_lock_reason&#x27;</span>, <span class="hljs-string">&#x27;pull_request&#x27;</span>, <span class="hljs-string">&#x27;body&#x27;</span>, <span class="hljs-string">&#x27;performed_via_github_app&#x27;</span>, <span class="hljs-string">&#x27;is_pull_request&#x27;</span>],
    num_rows: <span class="hljs-number">771</span>
})`}}),ds=new O({props:{code:`columns = issues_dataset.column_names
columns_to_keep = ["title", "body", "html_url", "comments"]
columns_to_remove = set(columns_to_keep).symmetric_difference(columns)
issues_dataset = issues_dataset.remove_columns(columns_to_remove)
issues_dataset`,highlighted:`columns = issues_dataset.column_names
columns_to_keep = [<span class="hljs-string">&quot;title&quot;</span>, <span class="hljs-string">&quot;body&quot;</span>, <span class="hljs-string">&quot;html_url&quot;</span>, <span class="hljs-string">&quot;comments&quot;</span>]
columns_to_remove = <span class="hljs-built_in">set</span>(columns_to_keep).symmetric_difference(columns)
issues_dataset = issues_dataset.remove_columns(columns_to_remove)
issues_dataset`}}),us=new O({props:{code:`Dataset({
    features: ['html_url', 'title', 'comments', 'body'],
    num_rows: 771
})`,highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;html_url&#x27;</span>, <span class="hljs-string">&#x27;title&#x27;</span>, <span class="hljs-string">&#x27;comments&#x27;</span>, <span class="hljs-string">&#x27;body&#x27;</span>],
    num_rows: <span class="hljs-number">771</span>
})`}}),cs=new O({props:{code:`issues_dataset.set_format("pandas")
df = issues_dataset[:]`,highlighted:`issues_dataset.set_format(<span class="hljs-string">&quot;pandas&quot;</span>)
df = issues_dataset[:]`}}),ms=new O({props:{code:'df["comments"][0].tolist()',highlighted:'df[<span class="hljs-string">&quot;comments&quot;</span>][<span class="hljs-number">0</span>].tolist()'}}),ps=new O({props:{code:`['the bug code locate in \uFF1A\\r\\n    if data_args.task_name is not None:\\r\\n        # Downloading and loading a dataset from the hub.\\r\\n        datasets = load_dataset("glue", data_args.task_name, cache_dir=model_args.cache_dir)',
 'Hi @jinec,\\r\\n\\r\\nFrom time to time we get this kind of \`ConnectionError\` coming from the github.com website: https://raw.githubusercontent.com\\r\\n\\r\\nNormally, it should work if you wait a little and then retry.\\r\\n\\r\\nCould you please confirm if the problem persists?',
 'cannot connect\uFF0Ceven by Web browser\uFF0Cplease check that  there is some  problems\u3002',
 'I can access https://raw.githubusercontent.com/huggingface/datasets/1.7.0/datasets/glue/glue.py without problem...']`,highlighted:`[<span class="hljs-string">&#x27;the bug code locate in \uFF1A\\r\\n    if data_args.task_name is not None:\\r\\n        # Downloading and loading a dataset from the hub.\\r\\n        datasets = load_dataset(&quot;glue&quot;, data_args.task_name, cache_dir=model_args.cache_dir)&#x27;</span>,
 <span class="hljs-string">&#x27;Hi @jinec,\\r\\n\\r\\nFrom time to time we get this kind of \`ConnectionError\` coming from the github.com website: https://raw.githubusercontent.com\\r\\n\\r\\nNormally, it should work if you wait a little and then retry.\\r\\n\\r\\nCould you please confirm if the problem persists?&#x27;</span>,
 <span class="hljs-string">&#x27;cannot connect\uFF0Ceven by Web browser\uFF0Cplease check that  there is some  problems\u3002&#x27;</span>,
 <span class="hljs-string">&#x27;I can access https://raw.githubusercontent.com/huggingface/datasets/1.7.0/datasets/glue/glue.py without problem...&#x27;</span>]`}}),fs=new O({props:{code:`comments_df = df.explode("comments", ignore_index=True)
comments_df.head(4)`,highlighted:`comments_df = df.explode(<span class="hljs-string">&quot;comments&quot;</span>, ignore_index=<span class="hljs-literal">True</span>)
comments_df.head(<span class="hljs-number">4</span>)`}}),hs=new O({props:{code:`from datasets import Dataset

comments_dataset = Dataset.from_pandas(comments_df)
comments_dataset`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset

comments_dataset = Dataset.from_pandas(comments_df)
comments_dataset`}}),_s=new O({props:{code:`Dataset({
    features: ['html_url', 'title', 'comments', 'body'],
    num_rows: 2842
})`,highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;html_url&#x27;</span>, <span class="hljs-string">&#x27;title&#x27;</span>, <span class="hljs-string">&#x27;comments&#x27;</span>, <span class="hljs-string">&#x27;body&#x27;</span>],
    num_rows: <span class="hljs-number">2842</span>
})`}}),ze=new Lu({props:{$$slots:{default:[Qu]},$$scope:{ctx:U}}}),gs=new O({props:{code:`comments_dataset = comments_dataset.map(
    lambda x: {"comment_length": len(x["comments"].split())}
)`,highlighted:`comments_dataset = comments_dataset.<span class="hljs-built_in">map</span>(
    <span class="hljs-keyword">lambda</span> x: {<span class="hljs-string">&quot;comment_length&quot;</span>: <span class="hljs-built_in">len</span>(x[<span class="hljs-string">&quot;comments&quot;</span>].split())}
)`}}),bs=new O({props:{code:`comments_dataset = comments_dataset.filter(lambda x: x["comment_length"] > 15)
comments_dataset`,highlighted:`comments_dataset = comments_dataset.<span class="hljs-built_in">filter</span>(<span class="hljs-keyword">lambda</span> x: x[<span class="hljs-string">&quot;comment_length&quot;</span>] &gt; <span class="hljs-number">15</span>)
comments_dataset`}}),vs=new O({props:{code:`Dataset({
    features: ['html_url', 'title', 'comments', 'body', 'comment_length'],
    num_rows: 2098
})`,highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;html_url&#x27;</span>, <span class="hljs-string">&#x27;title&#x27;</span>, <span class="hljs-string">&#x27;comments&#x27;</span>, <span class="hljs-string">&#x27;body&#x27;</span>, <span class="hljs-string">&#x27;comment_length&#x27;</span>],
    num_rows: <span class="hljs-number">2098</span>
})`}}),$s=new O({props:{code:`def concatenate_text(examples):
    return {
        "text": examples["title"]
        + " \\n "
        + examples["body"]
        + " \\n "
        + examples["comments"]
    }


comments_dataset = comments_dataset.map(concatenate_text)`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">concatenate_text</span>(<span class="hljs-params">examples</span>):
    <span class="hljs-keyword">return</span> {
        <span class="hljs-string">&quot;text&quot;</span>: examples[<span class="hljs-string">&quot;title&quot;</span>]
        + <span class="hljs-string">&quot; \\n &quot;</span>
        + examples[<span class="hljs-string">&quot;body&quot;</span>]
        + <span class="hljs-string">&quot; \\n &quot;</span>
        + examples[<span class="hljs-string">&quot;comments&quot;</span>]
    }


comments_dataset = comments_dataset.<span class="hljs-built_in">map</span>(concatenate_text)`}}),qs=new Pt({});const Fi=[Ku,Xu],Ss=[];function Ni(e,i){return e[0]==="pt"?0:1}ge=Ni(U),be=Ss[ge]=Fi[ge](U),ys=new O({props:{code:`def cls_pooling(model_output):
    return model_output.last_hidden_state[:, 0]`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">cls_pooling</span>(<span class="hljs-params">model_output</span>):
    <span class="hljs-keyword">return</span> model_output.last_hidden_state[:, <span class="hljs-number">0</span>]`}});const zi=[ec,Zu],Os=[];function Ui(e,i){return e[0]==="pt"?0:1}ve=Ui(U),$e=Os[ve]=zi[ve](U),ws=new Pt({}),js=new O({props:{code:'embeddings_dataset.add_faiss_index(column="embeddings")',highlighted:'embeddings_dataset.add_faiss_index(column=<span class="hljs-string">&quot;embeddings&quot;</span>)'}});const Gi=[ac,sc],Is=[];function Bi(e,i){return e[0]==="pt"?0:1}return qe=Bi(U),Ee=Is[qe]=Gi[qe](U),Ds=new O({props:{code:`scores, samples = embeddings_dataset.get_nearest_examples(
    "embeddings", question_embedding, k=5
)`,highlighted:`scores, samples = embeddings_dataset.get_nearest_examples(
    <span class="hljs-string">&quot;embeddings&quot;</span>, question_embedding, k=<span class="hljs-number">5</span>
)`}}),Ts=new O({props:{code:`import pandas as pd

samples_df = pd.DataFrame.from_dict(samples)
samples_df["scores"] = scores
samples_df.sort_values("scores", ascending=False, inplace=True)`,highlighted:`<span class="hljs-keyword">import</span> pandas <span class="hljs-keyword">as</span> pd

samples_df = pd.DataFrame.from_dict(samples)
samples_df[<span class="hljs-string">&quot;scores&quot;</span>] = scores
samples_df.sort_values(<span class="hljs-string">&quot;scores&quot;</span>, ascending=<span class="hljs-literal">False</span>, inplace=<span class="hljs-literal">True</span>)`}}),Cs=new O({props:{code:`for _, row in samples_df.iterrows():
    print(f"COMMENT: {row.comments}")
    print(f"SCORE: {row.scores}")
    print(f"TITLE: {row.title}")
    print(f"URL: {row.html_url}")
    print("=" * 50)
    print()`,highlighted:`<span class="hljs-keyword">for</span> _, row <span class="hljs-keyword">in</span> samples_df.iterrows():
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;COMMENT: <span class="hljs-subst">{row.comments}</span>&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;SCORE: <span class="hljs-subst">{row.scores}</span>&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;TITLE: <span class="hljs-subst">{row.title}</span>&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;URL: <span class="hljs-subst">{row.html_url}</span>&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;=&quot;</span> * <span class="hljs-number">50</span>)
    <span class="hljs-built_in">print</span>()`}}),As=new O({props:{code:`"""
COMMENT: Requiring online connection is a deal breaker in some cases unfortunately so it'd be great if offline mode is added similar to how \`transformers\` loads models offline fine.

@mandubian's second bullet point suggests that there's a workaround allowing you to use your offline (custom?) dataset with \`datasets\`. Could you please elaborate on how that should look like?
SCORE: 25.505046844482422
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: The local dataset builders (csv, text , json and pandas) are now part of the \`datasets\` package since #1726 :)
You can now use them offline
\\\`\\\`\\\`python
datasets = load_dataset("text", data_files=data_files)
\\\`\\\`\\\`

We'll do a new release soon
SCORE: 24.555509567260742
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: I opened a PR that allows to reload modules that have already been loaded once even if there's no internet.

Let me know if you know other ways that can make the offline mode experience better. I'd be happy to add them :)

I already note the "freeze" modules option, to prevent local modules updates. It would be a cool feature.

----------

> @mandubian's second bullet point suggests that there's a workaround allowing you to use your offline (custom?) dataset with \`datasets\`. Could you please elaborate on how that should look like?

Indeed \`load_dataset\` allows to load remote dataset script (squad, glue, etc.) but also you own local ones.
For example if you have a dataset script at \`./my_dataset/my_dataset.py\` then you can do
\\\`\\\`\\\`python
load_dataset("./my_dataset")
\\\`\\\`\\\`
and the dataset script will generate your dataset once and for all.

----------

About I'm looking into having \`csv\`, \`json\`, \`text\`, \`pandas\` dataset builders already included in the \`datasets\` package, so that they are available offline by default, as opposed to the other datasets that require the script to be downloaded.
cf #1724
SCORE: 24.14896583557129
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: > here is my way to load a dataset offline, but it **requires** an online machine
>
> 1. (online machine)
>
> \`\`\`
>
> import datasets
>
> data = datasets.load_dataset(...)
>
> data.save_to_disk(/YOUR/DATASET/DIR)
>
> \`\`\`
>
> 2. copy the dir from online to the offline machine
>
> 3. (offline machine)
>
> \`\`\`
>
> import datasets
>
> data = datasets.load_from_disk(/SAVED/DATA/DIR)
>
> \`\`\`
>
>
>
> HTH.


SCORE: 22.893993377685547
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: here is my way to load a dataset offline, but it **requires** an online machine
1. (online machine)
\\\`\\\`\\\`
import datasets
data = datasets.load_dataset(...)
data.save_to_disk(/YOUR/DATASET/DIR)
\\\`\\\`\\\`
2. copy the dir from online to the offline machine
3. (offline machine)
\\\`\\\`\\\`
import datasets
data = datasets.load_from_disk(/SAVED/DATA/DIR)
\\\`\\\`\\\`

HTH.
SCORE: 22.406635284423828
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================
"""`,highlighted:`<span class="hljs-string">&quot;&quot;&quot;
COMMENT: Requiring online connection is a deal breaker in some cases unfortunately so it&#x27;d be great if offline mode is added similar to how \`transformers\` loads models offline fine.

@mandubian&#x27;s second bullet point suggests that there&#x27;s a workaround allowing you to use your offline (custom?) dataset with \`datasets\`. Could you please elaborate on how that should look like?
SCORE: 25.505046844482422
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: The local dataset builders (csv, text , json and pandas) are now part of the \`datasets\` package since #1726 :)
You can now use them offline
\\\`\\\`\\\`python
datasets = load_dataset(&quot;text&quot;, data_files=data_files)
\\\`\\\`\\\`

We&#x27;ll do a new release soon
SCORE: 24.555509567260742
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: I opened a PR that allows to reload modules that have already been loaded once even if there&#x27;s no internet.

Let me know if you know other ways that can make the offline mode experience better. I&#x27;d be happy to add them :)

I already note the &quot;freeze&quot; modules option, to prevent local modules updates. It would be a cool feature.

----------

&gt; @mandubian&#x27;s second bullet point suggests that there&#x27;s a workaround allowing you to use your offline (custom?) dataset with \`datasets\`. Could you please elaborate on how that should look like?

Indeed \`load_dataset\` allows to load remote dataset script (squad, glue, etc.) but also you own local ones.
For example if you have a dataset script at \`./my_dataset/my_dataset.py\` then you can do
\\\`\\\`\\\`python
load_dataset(&quot;./my_dataset&quot;)
\\\`\\\`\\\`
and the dataset script will generate your dataset once and for all.

----------

About I&#x27;m looking into having \`csv\`, \`json\`, \`text\`, \`pandas\` dataset builders already included in the \`datasets\` package, so that they are available offline by default, as opposed to the other datasets that require the script to be downloaded.
cf #1724
SCORE: 24.14896583557129
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: &gt; here is my way to load a dataset offline, but it **requires** an online machine
&gt;
&gt; 1. (online machine)
&gt;
&gt; \`\`\`
&gt;
&gt; import datasets
&gt;
&gt; data = datasets.load_dataset(...)
&gt;
&gt; data.save_to_disk(/YOUR/DATASET/DIR)
&gt;
&gt; \`\`\`
&gt;
&gt; 2. copy the dir from online to the offline machine
&gt;
&gt; 3. (offline machine)
&gt;
&gt; \`\`\`
&gt;
&gt; import datasets
&gt;
&gt; data = datasets.load_from_disk(/SAVED/DATA/DIR)
&gt;
&gt; \`\`\`
&gt;
&gt;
&gt;
&gt; HTH.


SCORE: 22.893993377685547
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: here is my way to load a dataset offline, but it **requires** an online machine
1. (online machine)
\\\`\\\`\\\`
import datasets
data = datasets.load_dataset(...)
data.save_to_disk(/YOUR/DATASET/DIR)
\\\`\\\`\\\`
2. copy the dir from online to the offline machine
3. (offline machine)
\\\`\\\`\\\`
import datasets
data = datasets.load_from_disk(/SAVED/DATA/DIR)
\\\`\\\`\\\`

HTH.
SCORE: 22.406635284423828
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================
&quot;&quot;&quot;</span>`}}),Je=new Lu({props:{$$slots:{default:[tc]},$$scope:{ctx:U}}}),{c(){m=n("meta"),b=u(),q(f.$$.fragment),j=u(),h=n("h1"),v=n("a"),R=n("span"),q(C.$$.fragment),I=u(),w=n("span"),$=t("B\xFAsqueda sem\xE1ntica con FAISS"),D=u(),M.c(),H=u(),F=n("p"),B=t("En la "),A=n("a"),L=t("secci\xF3n 5"),W=t(" creamos un dataset de issues y comentarios del repositorio de Github de \u{1F917} Datasets. En esta secci\xF3n usaremos esta informaci\xF3n para construir un motor de b\xFAsqueda que nos ayude a responder nuestras preguntas m\xE1s apremiantes sobre la librer\xEDa."),G=u(),q(T.$$.fragment),Y=u(),p=n("h2"),S=n("a"),N=n("span"),q(J.$$.fragment),le=u(),he=n("span"),Zo=t("Usando _embeddings_ para la b\xFAsqueda sem\xE1ntica"),St=u(),Q=n("p"),en=t("Como vimos en el "),Hs=n("a"),sn=t("Cap\xEDtulo 1"),an=t(", los modelos de lenguaje basados en Transformers representan cada token en un texto como un "),ta=n("em"),tn=t("vector de embeddings"),on=t(". Resulta que podemos agrupar los "),oa=n("em"),nn=t("embeddings"),rn=t(" individuales en representaciones vectoriales para oraciones, p\xE1rrafos o (en algunos casos) documentos completos. Estos "),na=n("em"),ln=t("embeddings"),dn=t(" pueden ser usados para encontrar documentos similares en el corpus al calcular la similaridad del producto punto (o alguna otra m\xE9trica de similaridad) entre cada "),ra=n("em"),un=t("embedding"),cn=t(" y devolver los documentos con la mayor coincidencia."),Ot=u(),Re=n("p"),mn=t("En esta secci\xF3n vamos a usar "),la=n("em"),pn=t("embeddings"),fn=t(" para desarrollar un motor de b\xFAsqueda sem\xE1ntica. Estos motores de b\xFAsqueda tienen varias ventajas sobre abordajes convencionales basados en la coincidencia de palabras clave en una b\xFAsqueda con los documentos."),It=u(),Se=n("div"),ss=n("img"),hn=u(),as=n("img"),Mt=u(),Oe=n("h2"),He=n("a"),ia=n("span"),q(ts.$$.fragment),_n=u(),da=n("span"),gn=t("Cargando y preparando el dataset"),Rt=u(),Ls=n("p"),bn=t("Lo primero que tenemos que hacer es descargar el dataset de issues de GitHub, as\xED que usaremos la librer\xEDa \u{1F917} Hub para resolver la URL en la que est\xE1 almacenado nuestro archivo en el Hub de Hugging Face:"),Ht=u(),q(os.$$.fragment),Lt=u(),xe=n("p"),vn=t("Con la URL almacenada en "),ua=n("code"),$n=t("data_files"),qn=t(", podemos cargar el dataset remoto usando el m\xE9todo introducido en la "),Fs=n("a"),En=t("secci\xF3n 2"),xn=t(":"),Ft=u(),q(ns.$$.fragment),Nt=u(),q(rs.$$.fragment),zt=u(),X=n("p"),yn=t("Hemos especificado el conjunto "),ca=n("code"),wn=t("train"),kn=t(" por defecto en "),ma=n("code"),jn=t("load_dataset()"),Dn=t(", de tal manera que devuelva un objeto "),pa=n("code"),Tn=t("Dataset"),Cn=t(" en vez de un "),fa=n("code"),An=t("DatasetDict"),Pn=t(". Lo primero que debemos hacer es filtrar los pull requests, dado que estos no se suelen usar para resolver preguntas de usuarios e introducir\xE1n ruido en nuestro motor de b\xFAsqueda. Como ya debe ser familiar para ti, podemos usar la funci\xF3n "),ha=n("code"),Sn=t("Dataset.filter()"),On=t(" para excluir estas filas en nuestro dataset. A su vez, filtremos las filas que no tienen comentarios, dado que no van a darnos respuestas para las preguntas de los usuarios."),Ut=u(),q(ls.$$.fragment),Gt=u(),q(is.$$.fragment),Bt=u(),K=n("p"),In=t("Podemos ver que hay un gran n\xFAmero de columnas en nuestro dataset, muchas de las cuales no necesitamos para construir nuestro motor de b\xFAsqueda. Desde la perspectiva de la b\xFAsqueda, las columnas m\xE1s informativas son "),_a=n("code"),Mn=t("title"),Rn=t(", "),ga=n("code"),Hn=t("body"),Ln=t(" y "),ba=n("code"),Fn=t("comments"),Nn=t(", mientras que "),va=n("code"),zn=t("html_url"),Un=t(" nos indica un link al issue correspondiente. Usemos la funci\xF3n "),$a=n("code"),Gn=t("Dataset.remove_columns()"),Bn=t(" para eliminar el resto:"),Yt=u(),q(ds.$$.fragment),Vt=u(),q(us.$$.fragment),Wt=u(),Z=n("p"),Yn=t("Para crear nuestros "),qa=n("em"),Vn=t("embeddings"),Wn=t(", vamos a ampliar cada comentario a\xF1adi\xE9ndole el t\xEDtulo y el cuerpo del issue, dado que estos campos suelen incluir informaci\xF3n de contexto \xFAtil. Dado que nuestra funci\xF3n "),Ea=n("code"),Jn=t("comments"),Qn=t(" es una lista de comentarios para cada issue, necesitamos \u201Cexplotar\u201D la columna para que cada fila sea una tupla "),xa=n("code"),Xn=t("(html_url, title, body, comment)"),Kn=t(". Podemos hacer esto en Pandas con la "),Le=n("a"),Zn=t("funci\xF3n "),ya=n("code"),er=t("DataFrame.explode()"),sr=t(", que crea una nueva fila para cada elemento en una columna que est\xE1 en forma de lista, al tiempo que replica el resto de los valores de las otras columnas. Para verlo en acci\xF3n, primero debemos cambiar al formato "),wa=n("code"),ar=t("DataFrame"),tr=t(" de Pandas:"),Jt=u(),q(cs.$$.fragment),Qt=u(),Fe=n("p"),or=t("Si inspeccionamos la primera fila en este "),ka=n("code"),nr=t("DataFrame"),rr=t(" podemos ver que hay 4 comentarios asociados con este issue:"),Xt=u(),q(ms.$$.fragment),Kt=u(),q(ps.$$.fragment),Zt=u(),Ne=n("p"),lr=t("Cuando \u201Cexplotamos\u201D "),ja=n("code"),ir=t("df"),dr=t(", queremos obtener una fila para cada uno de estos comentarios. Veamos si este es el caso:"),eo=u(),q(fs.$$.fragment),so=u(),ae=n("table"),Da=n("thead"),ee=n("tr"),ao=n("th"),ur=u(),Ta=n("th"),cr=t("html_url"),mr=u(),Ca=n("th"),pr=t("title"),fr=u(),Aa=n("th"),hr=t("comments"),_r=u(),Pa=n("th"),gr=t("body"),br=u(),_e=n("tbody"),te=n("tr"),Sa=n("th"),vr=t("0"),$r=u(),Oa=n("td"),qr=t("https://github.com/huggingface/datasets/issues/2787"),Er=u(),Ia=n("td"),xr=t("ConnectionError: Couldn't reach https://raw.githubusercontent.com"),yr=u(),Ma=n("td"),wr=t("the bug code locate in \uFF1A\\r\\n    if data_args.task_name is not None..."),kr=u(),Ra=n("td"),jr=t("Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),Dr=u(),oe=n("tr"),Ha=n("th"),Tr=t("1"),Cr=u(),La=n("td"),Ar=t("https://github.com/huggingface/datasets/issues/2787"),Pr=u(),Fa=n("td"),Sr=t("ConnectionError: Couldn't reach https://raw.githubusercontent.com"),Or=u(),Na=n("td"),Ir=t("Hi @jinec,\\r\\n\\r\\nFrom time to time we get this kind of `ConnectionError` coming from the github.com website: https://raw.githubusercontent.com..."),Mr=u(),za=n("td"),Rr=t("Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),Hr=u(),ne=n("tr"),Ua=n("th"),Lr=t("2"),Fr=u(),Ga=n("td"),Nr=t("https://github.com/huggingface/datasets/issues/2787"),zr=u(),Ba=n("td"),Ur=t("ConnectionError: Couldn't reach https://raw.githubusercontent.com"),Gr=u(),Ya=n("td"),Br=t("cannot connect\uFF0Ceven by Web browser\uFF0Cplease check that  there is some  problems\u3002"),Yr=u(),Va=n("td"),Vr=t("Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),Wr=u(),re=n("tr"),Wa=n("th"),Jr=t("3"),Qr=u(),Ja=n("td"),Xr=t("https://github.com/huggingface/datasets/issues/2787"),Kr=u(),Qa=n("td"),Zr=t("ConnectionError: Couldn't reach https://raw.githubusercontent.com"),el=u(),Xa=n("td"),sl=t("I can access https://raw.githubusercontent.com/huggingface/datasets/1.7.0/datasets/glue/glue.py without problem..."),al=u(),Ka=n("td"),tl=t("Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),to=u(),ie=n("p"),ol=t("Genial, podemos ver que las filas se han replicado y que la columna "),Za=n("code"),nl=t("comments"),rl=t(" incluye los comentarios individuales. Ahora que hemos terminado con Pandas, podemos volver a cambiar el formato a "),et=n("code"),ll=t("Dataset"),il=t(" cargando el "),st=n("code"),dl=t("DataFrame"),ul=t(" en memoria:"),oo=u(),q(hs.$$.fragment),no=u(),q(_s.$$.fragment),ro=u(),Ns=n("p"),cl=t("\xA1Esto nos ha dado varios miles de comentarios con los que trabajar!"),lo=u(),q(ze.$$.fragment),io=u(),Ue=n("p"),ml=t("Ahora que tenemos un comentario para cada fila, creemos una columna "),at=n("code"),pl=t("comments_length"),fl=t(" que contenga el n\xFAmero de palabras por comentario:"),uo=u(),q(gs.$$.fragment),co=u(),zs=n("p"),hl=t("Podemos usar esta nueva columna para filtrar los comentarios cortos, que t\xEDpicamente incluyen cosas como \u201Ccc @letwun\u201D o \u201C\xA1Gracias!\u201D, que no son relevantes para nuestro motor de b\xFAsqueda. No hay un n\xFAmero preciso que debamos filtrar, pero alrededor de 15 palabras es un buen comienzo:"),mo=u(),q(bs.$$.fragment),po=u(),q(vs.$$.fragment),fo=u(),ye=n("p"),_l=t("Ahora que hemos limpiado un poco el dataset, vamos a concatenar el t\xEDtulo, la descripci\xF3n y los comentarios del issue en una nueva columna "),tt=n("code"),gl=t("text"),bl=t(". Como lo hemos venido haciendo, escribiremos una funci\xF3n para pasarla a "),ot=n("code"),vl=t("Dataset.map()"),$l=t(":"),ho=u(),q($s.$$.fragment),_o=u(),Ge=n("p"),ql=t("\xA1Por fin estamos listos para crear "),nt=n("em"),El=t("embeddings"),xl=t("!"),go=u(),Ie=n("h2"),Be=n("a"),rt=n("span"),q(qs.$$.fragment),yl=u(),lt=n("span"),wl=t("Creando _embeddings_ de texto"),bo=u(),z=n("p"),kl=t("En el "),Us=n("a"),jl=t("Cap\xEDtulo 2"),Dl=t(" vimos que podemos obtener "),it=n("em"),Tl=t("embeddings"),Cl=t(" usando la clase "),dt=n("code"),Al=t("AutoModel"),Pl=t(". Todo lo que tenemos que hacer es escoger un punto de control adecuado para cargar el modelo. Afortunadamente, existe una librer\xEDa llamada "),ut=n("code"),Sl=t("sentence-transformers"),Ol=t(" que se especializa en crear "),ct=n("em"),Il=t("embeddings"),Ml=t(". Como se describe en la "),Es=n("a"),Rl=t("documentaci\xF3n"),Hl=t(" de esta librer\xEDa, nuestro caso de uso es un ejemplo de "),mt=n("em"),Ll=t("b\xFAsqueda sem\xE1ntica asim\xE9trica"),Fl=t(" porque tenemos una pregunta corta cuya respuesta queremos encontrar en un documento m\xE1s grande, como un comentario de un issue. La tabla de "),xs=n("a"),Nl=t("resumen de modelos"),zl=t(" en la documentaci\xF3n nos indica que el punto de control "),pt=n("code"),Ul=t("multi-qa-mpnet-base-dot-v1"),Gl=t(" tiene el mejor desempe\xF1o para la b\xFAsqueda sem\xE1ntica, as\xED que lo usaremos para nuestra aplicaci\xF3n. Tambi\xE9n cargaremos el tokenizador usando el mismo punto de control:"),vo=u(),be.c(),Gs=u(),de=n("p"),Bl=t("Como mencionamos con anterioridad, queremos representar cada entrada en el corpus de issues de GitHub como un vector individual, as\xED que necesitamos agrupar o promediar nuestros "),ft=n("em"),Yl=t("embeddings"),Vl=t(" de tokes de alguna manera. Un abordaje popular es ejecutar "),ht=n("em"),Wl=t("CLS pooling"),Jl=t(" en los outputs de nuestro modelo, donde simplemente vamos a recolectar el \xFAltimo estado oculto para el token especial "),_t=n("code"),Ql=t("[CLS]"),Xl=t(". La siguiente funci\xF3n nos ayudar\xE1 con esto:"),$o=u(),q(ys.$$.fragment),qo=u(),Bs=n("p"),Kl=t("Ahora crearemos una funci\xF3n que va a tokenizar una lista de documentos, ubicar los tensores en la GPU, alimentarlos al modelo y aplicar CLS pooling a los outputs:"),Eo=u(),$e.c(),Ys=u(),Ye=n("p"),Zl=t("Los "),gt=n("em"),ei=t("embeddings"),si=t(" se han convertido en arrays de NumPy, esto es porque \u{1F917} Datasets los necesita en este formato cuando queremos indexarlos con FAISS, que es lo que haremos a continuaci\xF3n."),xo=u(),Me=n("h2"),Ve=n("a"),bt=n("span"),q(ws.$$.fragment),ai=u(),vt=n("span"),ti=t("Usando FAISS para una b\xFAsqueda eficiente por similaridad"),yo=u(),se=n("p"),oi=t("Ahora que tenemos un dataset de embeddings, necesitamos una manera de buscar sobre ellos. Para hacerlo, usaremos una estructura especial de datos en \u{1F917} Datasets llamada "),$t=n("em"),ni=t("\xEDndice FAISS"),ri=t(". [FAISS] ("),ks=n("a"),li=t("https://faiss.ai/"),ii=t(") (siglas para "),qt=n("em"),di=t("Facebook AI Similarity Search"),ui=t(") es una librer\xEDa que contiene algoritmos eficientes para buscar y agrupar r\xE1pidamente vectores de "),Et=n("em"),ci=t("embeddings"),mi=t("."),wo=u(),ue=n("p"),pi=t("La idea b\xE1sica detr\xE1s de FAISS es que crea una estructura especial de datos, llamada "),xt=n("em"),fi=t("\xEDndice"),hi=t(", que te permite encontrar cu\xE1les embeddings son parecidos a un "),yt=n("em"),_i=t("embedding"),gi=t(" de entrada. La creaci\xF3n de un \xEDndice FAISS en \u{1F917} Datasets es muy simple: usamos la funci\xF3n "),wt=n("code"),bi=t("Dataset.add_faiss_index()"),vi=t(" y especificamos cu\xE1l columna del dataset queremos indexar:"),ko=u(),q(js.$$.fragment),jo=u(),we=n("p"),$i=t("Ahora podemos hacer b\xFAsquedas sobre este \xEDndice al hacer una b\xFAsqueda del vecino m\xE1s cercano con la funci\xF3n "),kt=n("code"),qi=t("Dataset.get_nearest_examples()"),Ei=t(". Prob\xE9moslo al hacer el "),jt=n("em"),xi=t("embedding"),yi=t(" de una pregunta de la siguiente manera:"),Do=u(),Ee.c(),Vs=u(),We=n("p"),wi=t("Tal como en los documentos, ahora tenemos un vector de 768 dimensiones que representa la pregunta, que podemos comparar con el corpus entero para encontrar los "),Dt=n("em"),ki=t("embeddings"),ji=t(" m\xE1s parecidos:"),To=u(),q(Ds.$$.fragment),Co=u(),ke=n("p"),Di=t("La funci\xF3n "),Tt=n("code"),Ti=t("Dataset.get_nearest_examples()"),Ci=t(" devuelve una tupla de puntajes que calcula un ranking de la coincidencia entre la pregunta y el documento, as\xED como un conjunto correspondiente de muestras (en este caso, los 5 mejores resultados). Recoj\xE1moslos en un "),Ct=n("code"),Ai=t("pandas.DataFrame"),Pi=t(" para ordenarlos f\xE1cilmente:"),Ao=u(),q(Ts.$$.fragment),Po=u(),Ws=n("p"),Si=t("Podemos iterar sobre las primeras filas para ver qu\xE9 tanto coincide la pregunta con los comentarios disponibles:"),So=u(),q(Cs.$$.fragment),Oo=u(),q(As.$$.fragment),Io=u(),Js=n("p"),Oi=t("\xA1No est\xE1 mal! El segundo comentario parece responder la pregunta."),Mo=u(),q(Je.$$.fragment),this.h()},l(e){const i=Gu('[data-svelte="svelte-1phssyn"]',document.head);m=r(i,"META",{name:!0,content:!0}),i.forEach(a),b=c(e),E(f.$$.fragment,e),j=c(e),h=r(e,"H1",{class:!0});var Ms=l(h);v=r(Ms,"A",{id:!0,class:!0,href:!0});var Qs=l(v);R=r(Qs,"SPAN",{});var At=l(R);E(C.$$.fragment,At),At.forEach(a),Qs.forEach(a),I=c(Ms),w=r(Ms,"SPAN",{});var Xs=l(w);$=o(Xs,"B\xFAsqueda sem\xE1ntica con FAISS"),Xs.forEach(a),Ms.forEach(a),D=c(e),M.l(e),H=c(e),F=r(e,"P",{});var Qe=l(F);B=o(Qe,"En la "),A=r(Qe,"A",{href:!0});var Ks=l(A);L=o(Ks,"secci\xF3n 5"),Ks.forEach(a),W=o(Qe," creamos un dataset de issues y comentarios del repositorio de Github de \u{1F917} Datasets. En esta secci\xF3n usaremos esta informaci\xF3n para construir un motor de b\xFAsqueda que nos ayude a responder nuestras preguntas m\xE1s apremiantes sobre la librer\xEDa."),Qe.forEach(a),G=c(e),E(T.$$.fragment,e),Y=c(e),p=r(e,"H2",{class:!0});var Rs=l(p);S=r(Rs,"A",{id:!0,class:!0,href:!0});var Yi=l(S);N=r(Yi,"SPAN",{});var Vi=l(N);E(J.$$.fragment,Vi),Vi.forEach(a),Yi.forEach(a),le=c(Rs),he=r(Rs,"SPAN",{});var Wi=l(he);Zo=o(Wi,"Usando _embeddings_ para la b\xFAsqueda sem\xE1ntica"),Wi.forEach(a),Rs.forEach(a),St=c(e),Q=r(e,"P",{});var ce=l(Q);en=o(ce,"Como vimos en el "),Hs=r(ce,"A",{href:!0});var Ji=l(Hs);sn=o(Ji,"Cap\xEDtulo 1"),Ji.forEach(a),an=o(ce,", los modelos de lenguaje basados en Transformers representan cada token en un texto como un "),ta=r(ce,"EM",{});var Qi=l(ta);tn=o(Qi,"vector de embeddings"),Qi.forEach(a),on=o(ce,". Resulta que podemos agrupar los "),oa=r(ce,"EM",{});var Xi=l(oa);nn=o(Xi,"embeddings"),Xi.forEach(a),rn=o(ce," individuales en representaciones vectoriales para oraciones, p\xE1rrafos o (en algunos casos) documentos completos. Estos "),na=r(ce,"EM",{});var Ki=l(na);ln=o(Ki,"embeddings"),Ki.forEach(a),dn=o(ce," pueden ser usados para encontrar documentos similares en el corpus al calcular la similaridad del producto punto (o alguna otra m\xE9trica de similaridad) entre cada "),ra=r(ce,"EM",{});var Zi=l(ra);un=o(Zi,"embedding"),Zi.forEach(a),cn=o(ce," y devolver los documentos con la mayor coincidencia."),ce.forEach(a),Ot=c(e),Re=r(e,"P",{});var Ho=l(Re);mn=o(Ho,"En esta secci\xF3n vamos a usar "),la=r(Ho,"EM",{});var ed=l(la);pn=o(ed,"embeddings"),ed.forEach(a),fn=o(Ho," para desarrollar un motor de b\xFAsqueda sem\xE1ntica. Estos motores de b\xFAsqueda tienen varias ventajas sobre abordajes convencionales basados en la coincidencia de palabras clave en una b\xFAsqueda con los documentos."),Ho.forEach(a),It=c(e),Se=r(e,"DIV",{class:!0});var Lo=l(Se);ss=r(Lo,"IMG",{class:!0,src:!0,alt:!0}),hn=c(Lo),as=r(Lo,"IMG",{class:!0,src:!0,alt:!0}),Lo.forEach(a),Mt=c(e),Oe=r(e,"H2",{class:!0});var Fo=l(Oe);He=r(Fo,"A",{id:!0,class:!0,href:!0});var sd=l(He);ia=r(sd,"SPAN",{});var ad=l(ia);E(ts.$$.fragment,ad),ad.forEach(a),sd.forEach(a),_n=c(Fo),da=r(Fo,"SPAN",{});var td=l(da);gn=o(td,"Cargando y preparando el dataset"),td.forEach(a),Fo.forEach(a),Rt=c(e),Ls=r(e,"P",{});var od=l(Ls);bn=o(od,"Lo primero que tenemos que hacer es descargar el dataset de issues de GitHub, as\xED que usaremos la librer\xEDa \u{1F917} Hub para resolver la URL en la que est\xE1 almacenado nuestro archivo en el Hub de Hugging Face:"),od.forEach(a),Ht=c(e),E(os.$$.fragment,e),Lt=c(e),xe=r(e,"P",{});var Zs=l(xe);vn=o(Zs,"Con la URL almacenada en "),ua=r(Zs,"CODE",{});var nd=l(ua);$n=o(nd,"data_files"),nd.forEach(a),qn=o(Zs,", podemos cargar el dataset remoto usando el m\xE9todo introducido en la "),Fs=r(Zs,"A",{href:!0});var rd=l(Fs);En=o(rd,"secci\xF3n 2"),rd.forEach(a),xn=o(Zs,":"),Zs.forEach(a),Ft=c(e),E(ns.$$.fragment,e),Nt=c(e),E(rs.$$.fragment,e),zt=c(e),X=r(e,"P",{});var me=l(X);yn=o(me,"Hemos especificado el conjunto "),ca=r(me,"CODE",{});var ld=l(ca);wn=o(ld,"train"),ld.forEach(a),kn=o(me," por defecto en "),ma=r(me,"CODE",{});var id=l(ma);jn=o(id,"load_dataset()"),id.forEach(a),Dn=o(me,", de tal manera que devuelva un objeto "),pa=r(me,"CODE",{});var dd=l(pa);Tn=o(dd,"Dataset"),dd.forEach(a),Cn=o(me," en vez de un "),fa=r(me,"CODE",{});var ud=l(fa);An=o(ud,"DatasetDict"),ud.forEach(a),Pn=o(me,". Lo primero que debemos hacer es filtrar los pull requests, dado que estos no se suelen usar para resolver preguntas de usuarios e introducir\xE1n ruido en nuestro motor de b\xFAsqueda. Como ya debe ser familiar para ti, podemos usar la funci\xF3n "),ha=r(me,"CODE",{});var cd=l(ha);Sn=o(cd,"Dataset.filter()"),cd.forEach(a),On=o(me," para excluir estas filas en nuestro dataset. A su vez, filtremos las filas que no tienen comentarios, dado que no van a darnos respuestas para las preguntas de los usuarios."),me.forEach(a),Ut=c(e),E(ls.$$.fragment,e),Gt=c(e),E(is.$$.fragment,e),Bt=c(e),K=r(e,"P",{});var pe=l(K);In=o(pe,"Podemos ver que hay un gran n\xFAmero de columnas en nuestro dataset, muchas de las cuales no necesitamos para construir nuestro motor de b\xFAsqueda. Desde la perspectiva de la b\xFAsqueda, las columnas m\xE1s informativas son "),_a=r(pe,"CODE",{});var md=l(_a);Mn=o(md,"title"),md.forEach(a),Rn=o(pe,", "),ga=r(pe,"CODE",{});var pd=l(ga);Hn=o(pd,"body"),pd.forEach(a),Ln=o(pe," y "),ba=r(pe,"CODE",{});var fd=l(ba);Fn=o(fd,"comments"),fd.forEach(a),Nn=o(pe,", mientras que "),va=r(pe,"CODE",{});var hd=l(va);zn=o(hd,"html_url"),hd.forEach(a),Un=o(pe," nos indica un link al issue correspondiente. Usemos la funci\xF3n "),$a=r(pe,"CODE",{});var _d=l($a);Gn=o(_d,"Dataset.remove_columns()"),_d.forEach(a),Bn=o(pe," para eliminar el resto:"),pe.forEach(a),Yt=c(e),E(ds.$$.fragment,e),Vt=c(e),E(us.$$.fragment,e),Wt=c(e),Z=r(e,"P",{});var fe=l(Z);Yn=o(fe,"Para crear nuestros "),qa=r(fe,"EM",{});var gd=l(qa);Vn=o(gd,"embeddings"),gd.forEach(a),Wn=o(fe,", vamos a ampliar cada comentario a\xF1adi\xE9ndole el t\xEDtulo y el cuerpo del issue, dado que estos campos suelen incluir informaci\xF3n de contexto \xFAtil. Dado que nuestra funci\xF3n "),Ea=r(fe,"CODE",{});var bd=l(Ea);Jn=o(bd,"comments"),bd.forEach(a),Qn=o(fe," es una lista de comentarios para cada issue, necesitamos \u201Cexplotar\u201D la columna para que cada fila sea una tupla "),xa=r(fe,"CODE",{});var vd=l(xa);Xn=o(vd,"(html_url, title, body, comment)"),vd.forEach(a),Kn=o(fe,". Podemos hacer esto en Pandas con la "),Le=r(fe,"A",{href:!0,rel:!0});var Ii=l(Le);Zn=o(Ii,"funci\xF3n "),ya=r(Ii,"CODE",{});var $d=l(ya);er=o($d,"DataFrame.explode()"),$d.forEach(a),Ii.forEach(a),sr=o(fe,", que crea una nueva fila para cada elemento en una columna que est\xE1 en forma de lista, al tiempo que replica el resto de los valores de las otras columnas. Para verlo en acci\xF3n, primero debemos cambiar al formato "),wa=r(fe,"CODE",{});var qd=l(wa);ar=o(qd,"DataFrame"),qd.forEach(a),tr=o(fe," de Pandas:"),fe.forEach(a),Jt=c(e),E(cs.$$.fragment,e),Qt=c(e),Fe=r(e,"P",{});var No=l(Fe);or=o(No,"Si inspeccionamos la primera fila en este "),ka=r(No,"CODE",{});var Ed=l(ka);nr=o(Ed,"DataFrame"),Ed.forEach(a),rr=o(No," podemos ver que hay 4 comentarios asociados con este issue:"),No.forEach(a),Xt=c(e),E(ms.$$.fragment,e),Kt=c(e),E(ps.$$.fragment,e),Zt=c(e),Ne=r(e,"P",{});var zo=l(Ne);lr=o(zo,"Cuando \u201Cexplotamos\u201D "),ja=r(zo,"CODE",{});var xd=l(ja);ir=o(xd,"df"),xd.forEach(a),dr=o(zo,", queremos obtener una fila para cada uno de estos comentarios. Veamos si este es el caso:"),zo.forEach(a),eo=c(e),E(fs.$$.fragment,e),so=c(e),ae=r(e,"TABLE",{border:!0,class:!0,style:!0});var Uo=l(ae);Da=r(Uo,"THEAD",{});var yd=l(Da);ee=r(yd,"TR",{style:!0});var je=l(ee);ao=r(je,"TH",{}),l(ao).forEach(a),ur=c(je),Ta=r(je,"TH",{});var wd=l(Ta);cr=o(wd,"html_url"),wd.forEach(a),mr=c(je),Ca=r(je,"TH",{});var kd=l(Ca);pr=o(kd,"title"),kd.forEach(a),fr=c(je),Aa=r(je,"TH",{});var jd=l(Aa);hr=o(jd,"comments"),jd.forEach(a),_r=c(je),Pa=r(je,"TH",{});var Dd=l(Pa);gr=o(Dd,"body"),Dd.forEach(a),je.forEach(a),yd.forEach(a),br=c(Uo),_e=r(Uo,"TBODY",{});var Xe=l(_e);te=r(Xe,"TR",{});var De=l(te);Sa=r(De,"TH",{});var Td=l(Sa);vr=o(Td,"0"),Td.forEach(a),$r=c(De),Oa=r(De,"TD",{});var Cd=l(Oa);qr=o(Cd,"https://github.com/huggingface/datasets/issues/2787"),Cd.forEach(a),Er=c(De),Ia=r(De,"TD",{});var Ad=l(Ia);xr=o(Ad,"ConnectionError: Couldn't reach https://raw.githubusercontent.com"),Ad.forEach(a),yr=c(De),Ma=r(De,"TD",{});var Pd=l(Ma);wr=o(Pd,"the bug code locate in \uFF1A\\r\\n    if data_args.task_name is not None..."),Pd.forEach(a),kr=c(De),Ra=r(De,"TD",{});var Sd=l(Ra);jr=o(Sd,"Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),Sd.forEach(a),De.forEach(a),Dr=c(Xe),oe=r(Xe,"TR",{});var Te=l(oe);Ha=r(Te,"TH",{});var Od=l(Ha);Tr=o(Od,"1"),Od.forEach(a),Cr=c(Te),La=r(Te,"TD",{});var Id=l(La);Ar=o(Id,"https://github.com/huggingface/datasets/issues/2787"),Id.forEach(a),Pr=c(Te),Fa=r(Te,"TD",{});var Md=l(Fa);Sr=o(Md,"ConnectionError: Couldn't reach https://raw.githubusercontent.com"),Md.forEach(a),Or=c(Te),Na=r(Te,"TD",{});var Rd=l(Na);Ir=o(Rd,"Hi @jinec,\\r\\n\\r\\nFrom time to time we get this kind of `ConnectionError` coming from the github.com website: https://raw.githubusercontent.com..."),Rd.forEach(a),Mr=c(Te),za=r(Te,"TD",{});var Hd=l(za);Rr=o(Hd,"Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),Hd.forEach(a),Te.forEach(a),Hr=c(Xe),ne=r(Xe,"TR",{});var Ce=l(ne);Ua=r(Ce,"TH",{});var Ld=l(Ua);Lr=o(Ld,"2"),Ld.forEach(a),Fr=c(Ce),Ga=r(Ce,"TD",{});var Fd=l(Ga);Nr=o(Fd,"https://github.com/huggingface/datasets/issues/2787"),Fd.forEach(a),zr=c(Ce),Ba=r(Ce,"TD",{});var Nd=l(Ba);Ur=o(Nd,"ConnectionError: Couldn't reach https://raw.githubusercontent.com"),Nd.forEach(a),Gr=c(Ce),Ya=r(Ce,"TD",{});var zd=l(Ya);Br=o(zd,"cannot connect\uFF0Ceven by Web browser\uFF0Cplease check that  there is some  problems\u3002"),zd.forEach(a),Yr=c(Ce),Va=r(Ce,"TD",{});var Ud=l(Va);Vr=o(Ud,"Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),Ud.forEach(a),Ce.forEach(a),Wr=c(Xe),re=r(Xe,"TR",{});var Ae=l(re);Wa=r(Ae,"TH",{});var Gd=l(Wa);Jr=o(Gd,"3"),Gd.forEach(a),Qr=c(Ae),Ja=r(Ae,"TD",{});var Bd=l(Ja);Xr=o(Bd,"https://github.com/huggingface/datasets/issues/2787"),Bd.forEach(a),Kr=c(Ae),Qa=r(Ae,"TD",{});var Yd=l(Qa);Zr=o(Yd,"ConnectionError: Couldn't reach https://raw.githubusercontent.com"),Yd.forEach(a),el=c(Ae),Xa=r(Ae,"TD",{});var Vd=l(Xa);sl=o(Vd,"I can access https://raw.githubusercontent.com/huggingface/datasets/1.7.0/datasets/glue/glue.py without problem..."),Vd.forEach(a),al=c(Ae),Ka=r(Ae,"TD",{});var Wd=l(Ka);tl=o(Wd,"Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),Wd.forEach(a),Ae.forEach(a),Xe.forEach(a),Uo.forEach(a),to=c(e),ie=r(e,"P",{});var Ke=l(ie);ol=o(Ke,"Genial, podemos ver que las filas se han replicado y que la columna "),Za=r(Ke,"CODE",{});var Jd=l(Za);nl=o(Jd,"comments"),Jd.forEach(a),rl=o(Ke," incluye los comentarios individuales. Ahora que hemos terminado con Pandas, podemos volver a cambiar el formato a "),et=r(Ke,"CODE",{});var Qd=l(et);ll=o(Qd,"Dataset"),Qd.forEach(a),il=o(Ke," cargando el "),st=r(Ke,"CODE",{});var Xd=l(st);dl=o(Xd,"DataFrame"),Xd.forEach(a),ul=o(Ke," en memoria:"),Ke.forEach(a),oo=c(e),E(hs.$$.fragment,e),no=c(e),E(_s.$$.fragment,e),ro=c(e),Ns=r(e,"P",{});var Kd=l(Ns);cl=o(Kd,"\xA1Esto nos ha dado varios miles de comentarios con los que trabajar!"),Kd.forEach(a),lo=c(e),E(ze.$$.fragment,e),io=c(e),Ue=r(e,"P",{});var Go=l(Ue);ml=o(Go,"Ahora que tenemos un comentario para cada fila, creemos una columna "),at=r(Go,"CODE",{});var Zd=l(at);pl=o(Zd,"comments_length"),Zd.forEach(a),fl=o(Go," que contenga el n\xFAmero de palabras por comentario:"),Go.forEach(a),uo=c(e),E(gs.$$.fragment,e),co=c(e),zs=r(e,"P",{});var eu=l(zs);hl=o(eu,"Podemos usar esta nueva columna para filtrar los comentarios cortos, que t\xEDpicamente incluyen cosas como \u201Ccc @letwun\u201D o \u201C\xA1Gracias!\u201D, que no son relevantes para nuestro motor de b\xFAsqueda. No hay un n\xFAmero preciso que debamos filtrar, pero alrededor de 15 palabras es un buen comienzo:"),eu.forEach(a),mo=c(e),E(bs.$$.fragment,e),po=c(e),E(vs.$$.fragment,e),fo=c(e),ye=r(e,"P",{});var ea=l(ye);_l=o(ea,"Ahora que hemos limpiado un poco el dataset, vamos a concatenar el t\xEDtulo, la descripci\xF3n y los comentarios del issue en una nueva columna "),tt=r(ea,"CODE",{});var su=l(tt);gl=o(su,"text"),su.forEach(a),bl=o(ea,". Como lo hemos venido haciendo, escribiremos una funci\xF3n para pasarla a "),ot=r(ea,"CODE",{});var au=l(ot);vl=o(au,"Dataset.map()"),au.forEach(a),$l=o(ea,":"),ea.forEach(a),ho=c(e),E($s.$$.fragment,e),_o=c(e),Ge=r(e,"P",{});var Bo=l(Ge);ql=o(Bo,"\xA1Por fin estamos listos para crear "),nt=r(Bo,"EM",{});var tu=l(nt);El=o(tu,"embeddings"),tu.forEach(a),xl=o(Bo,"!"),Bo.forEach(a),go=c(e),Ie=r(e,"H2",{class:!0});var Yo=l(Ie);Be=r(Yo,"A",{id:!0,class:!0,href:!0});var ou=l(Be);rt=r(ou,"SPAN",{});var nu=l(rt);E(qs.$$.fragment,nu),nu.forEach(a),ou.forEach(a),yl=c(Yo),lt=r(Yo,"SPAN",{});var ru=l(lt);wl=o(ru,"Creando _embeddings_ de texto"),ru.forEach(a),Yo.forEach(a),bo=c(e),z=r(e,"P",{});var V=l(z);kl=o(V,"En el "),Us=r(V,"A",{href:!0});var lu=l(Us);jl=o(lu,"Cap\xEDtulo 2"),lu.forEach(a),Dl=o(V," vimos que podemos obtener "),it=r(V,"EM",{});var iu=l(it);Tl=o(iu,"embeddings"),iu.forEach(a),Cl=o(V," usando la clase "),dt=r(V,"CODE",{});var du=l(dt);Al=o(du,"AutoModel"),du.forEach(a),Pl=o(V,". Todo lo que tenemos que hacer es escoger un punto de control adecuado para cargar el modelo. Afortunadamente, existe una librer\xEDa llamada "),ut=r(V,"CODE",{});var uu=l(ut);Sl=o(uu,"sentence-transformers"),uu.forEach(a),Ol=o(V," que se especializa en crear "),ct=r(V,"EM",{});var cu=l(ct);Il=o(cu,"embeddings"),cu.forEach(a),Ml=o(V,". Como se describe en la "),Es=r(V,"A",{href:!0,rel:!0});var mu=l(Es);Rl=o(mu,"documentaci\xF3n"),mu.forEach(a),Hl=o(V," de esta librer\xEDa, nuestro caso de uso es un ejemplo de "),mt=r(V,"EM",{});var pu=l(mt);Ll=o(pu,"b\xFAsqueda sem\xE1ntica asim\xE9trica"),pu.forEach(a),Fl=o(V," porque tenemos una pregunta corta cuya respuesta queremos encontrar en un documento m\xE1s grande, como un comentario de un issue. La tabla de "),xs=r(V,"A",{href:!0,rel:!0});var fu=l(xs);Nl=o(fu,"resumen de modelos"),fu.forEach(a),zl=o(V," en la documentaci\xF3n nos indica que el punto de control "),pt=r(V,"CODE",{});var hu=l(pt);Ul=o(hu,"multi-qa-mpnet-base-dot-v1"),hu.forEach(a),Gl=o(V," tiene el mejor desempe\xF1o para la b\xFAsqueda sem\xE1ntica, as\xED que lo usaremos para nuestra aplicaci\xF3n. Tambi\xE9n cargaremos el tokenizador usando el mismo punto de control:"),V.forEach(a),vo=c(e),be.l(e),Gs=c(e),de=r(e,"P",{});var Ze=l(de);Bl=o(Ze,"Como mencionamos con anterioridad, queremos representar cada entrada en el corpus de issues de GitHub como un vector individual, as\xED que necesitamos agrupar o promediar nuestros "),ft=r(Ze,"EM",{});var _u=l(ft);Yl=o(_u,"embeddings"),_u.forEach(a),Vl=o(Ze," de tokes de alguna manera. Un abordaje popular es ejecutar "),ht=r(Ze,"EM",{});var gu=l(ht);Wl=o(gu,"CLS pooling"),gu.forEach(a),Jl=o(Ze," en los outputs de nuestro modelo, donde simplemente vamos a recolectar el \xFAltimo estado oculto para el token especial "),_t=r(Ze,"CODE",{});var bu=l(_t);Ql=o(bu,"[CLS]"),bu.forEach(a),Xl=o(Ze,". La siguiente funci\xF3n nos ayudar\xE1 con esto:"),Ze.forEach(a),$o=c(e),E(ys.$$.fragment,e),qo=c(e),Bs=r(e,"P",{});var vu=l(Bs);Kl=o(vu,"Ahora crearemos una funci\xF3n que va a tokenizar una lista de documentos, ubicar los tensores en la GPU, alimentarlos al modelo y aplicar CLS pooling a los outputs:"),vu.forEach(a),Eo=c(e),$e.l(e),Ys=c(e),Ye=r(e,"P",{});var Vo=l(Ye);Zl=o(Vo,"Los "),gt=r(Vo,"EM",{});var $u=l(gt);ei=o($u,"embeddings"),$u.forEach(a),si=o(Vo," se han convertido en arrays de NumPy, esto es porque \u{1F917} Datasets los necesita en este formato cuando queremos indexarlos con FAISS, que es lo que haremos a continuaci\xF3n."),Vo.forEach(a),xo=c(e),Me=r(e,"H2",{class:!0});var Wo=l(Me);Ve=r(Wo,"A",{id:!0,class:!0,href:!0});var qu=l(Ve);bt=r(qu,"SPAN",{});var Eu=l(bt);E(ws.$$.fragment,Eu),Eu.forEach(a),qu.forEach(a),ai=c(Wo),vt=r(Wo,"SPAN",{});var xu=l(vt);ti=o(xu,"Usando FAISS para una b\xFAsqueda eficiente por similaridad"),xu.forEach(a),Wo.forEach(a),yo=c(e),se=r(e,"P",{});var Pe=l(se);oi=o(Pe,"Ahora que tenemos un dataset de embeddings, necesitamos una manera de buscar sobre ellos. Para hacerlo, usaremos una estructura especial de datos en \u{1F917} Datasets llamada "),$t=r(Pe,"EM",{});var yu=l($t);ni=o(yu,"\xEDndice FAISS"),yu.forEach(a),ri=o(Pe,". [FAISS] ("),ks=r(Pe,"A",{href:!0,rel:!0});var wu=l(ks);li=o(wu,"https://faiss.ai/"),wu.forEach(a),ii=o(Pe,") (siglas para "),qt=r(Pe,"EM",{});var ku=l(qt);di=o(ku,"Facebook AI Similarity Search"),ku.forEach(a),ui=o(Pe,") es una librer\xEDa que contiene algoritmos eficientes para buscar y agrupar r\xE1pidamente vectores de "),Et=r(Pe,"EM",{});var ju=l(Et);ci=o(ju,"embeddings"),ju.forEach(a),mi=o(Pe,"."),Pe.forEach(a),wo=c(e),ue=r(e,"P",{});var es=l(ue);pi=o(es,"La idea b\xE1sica detr\xE1s de FAISS es que crea una estructura especial de datos, llamada "),xt=r(es,"EM",{});var Du=l(xt);fi=o(Du,"\xEDndice"),Du.forEach(a),hi=o(es,", que te permite encontrar cu\xE1les embeddings son parecidos a un "),yt=r(es,"EM",{});var Tu=l(yt);_i=o(Tu,"embedding"),Tu.forEach(a),gi=o(es," de entrada. La creaci\xF3n de un \xEDndice FAISS en \u{1F917} Datasets es muy simple: usamos la funci\xF3n "),wt=r(es,"CODE",{});var Cu=l(wt);bi=o(Cu,"Dataset.add_faiss_index()"),Cu.forEach(a),vi=o(es," y especificamos cu\xE1l columna del dataset queremos indexar:"),es.forEach(a),ko=c(e),E(js.$$.fragment,e),jo=c(e),we=r(e,"P",{});var sa=l(we);$i=o(sa,"Ahora podemos hacer b\xFAsquedas sobre este \xEDndice al hacer una b\xFAsqueda del vecino m\xE1s cercano con la funci\xF3n "),kt=r(sa,"CODE",{});var Au=l(kt);qi=o(Au,"Dataset.get_nearest_examples()"),Au.forEach(a),Ei=o(sa,". Prob\xE9moslo al hacer el "),jt=r(sa,"EM",{});var Pu=l(jt);xi=o(Pu,"embedding"),Pu.forEach(a),yi=o(sa," de una pregunta de la siguiente manera:"),sa.forEach(a),Do=c(e),Ee.l(e),Vs=c(e),We=r(e,"P",{});var Jo=l(We);wi=o(Jo,"Tal como en los documentos, ahora tenemos un vector de 768 dimensiones que representa la pregunta, que podemos comparar con el corpus entero para encontrar los "),Dt=r(Jo,"EM",{});var Su=l(Dt);ki=o(Su,"embeddings"),Su.forEach(a),ji=o(Jo," m\xE1s parecidos:"),Jo.forEach(a),To=c(e),E(Ds.$$.fragment,e),Co=c(e),ke=r(e,"P",{});var aa=l(ke);Di=o(aa,"La funci\xF3n "),Tt=r(aa,"CODE",{});var Ou=l(Tt);Ti=o(Ou,"Dataset.get_nearest_examples()"),Ou.forEach(a),Ci=o(aa," devuelve una tupla de puntajes que calcula un ranking de la coincidencia entre la pregunta y el documento, as\xED como un conjunto correspondiente de muestras (en este caso, los 5 mejores resultados). Recoj\xE1moslos en un "),Ct=r(aa,"CODE",{});var Iu=l(Ct);Ai=o(Iu,"pandas.DataFrame"),Iu.forEach(a),Pi=o(aa," para ordenarlos f\xE1cilmente:"),aa.forEach(a),Ao=c(e),E(Ts.$$.fragment,e),Po=c(e),Ws=r(e,"P",{});var Mu=l(Ws);Si=o(Mu,"Podemos iterar sobre las primeras filas para ver qu\xE9 tanto coincide la pregunta con los comentarios disponibles:"),Mu.forEach(a),So=c(e),E(Cs.$$.fragment,e),Oo=c(e),E(As.$$.fragment,e),Io=c(e),Js=r(e,"P",{});var Ru=l(Js);Oi=o(Ru,"\xA1No est\xE1 mal! El segundo comentario parece responder la pregunta."),Ru.forEach(a),Mo=c(e),E(Je.$$.fragment,e),this.h()},h(){k(m,"name","hf:doc:metadata"),k(m,"content",JSON.stringify(nc)),k(v,"id","bsqueda-semntica-con-faiss"),k(v,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),k(v,"href","#bsqueda-semntica-con-faiss"),k(h,"class","relative group"),k(A,"href","/course/chapter5/5"),k(S,"id","usando-embeddings-para-la-bsqueda-semntica"),k(S,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),k(S,"href","#usando-embeddings-para-la-bsqueda-semntica"),k(p,"class","relative group"),k(Hs,"href","/course/chapter1"),k(ss,"class","block dark:hidden"),Hu(ss.src,Mi="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter5/semantic-search.svg")||k(ss,"src",Mi),k(ss,"alt","Semantic search."),k(as,"class","hidden dark:block"),Hu(as.src,Ri="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter5/semantic-search-dark.svg")||k(as,"src",Ri),k(as,"alt","Semantic search."),k(Se,"class","flex justify-center"),k(He,"id","cargando-y-preparando-el-dataset"),k(He,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),k(He,"href","#cargando-y-preparando-el-dataset"),k(Oe,"class","relative group"),k(Fs,"href","/course/chapter5/2"),k(Le,"href","https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.explode.html"),k(Le,"rel","nofollow"),Qo(ee,"text-align","right"),k(ae,"border","1"),k(ae,"class","dataframe"),Qo(ae,"table-layout","fixed"),Qo(ae,"word-wrap","break-word"),Qo(ae,"width","100%"),k(Be,"id","creando-embeddings-de-texto"),k(Be,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),k(Be,"href","#creando-embeddings-de-texto"),k(Ie,"class","relative group"),k(Us,"href","/course/chapter2"),k(Es,"href","https://www.sbert.net/examples/applications/semantic-search/README.html#symmetric-vs-asymmetric-semantic-search"),k(Es,"rel","nofollow"),k(xs,"href","https://www.sbert.net/docs/pretrained_models.html#model-overview"),k(xs,"rel","nofollow"),k(Ve,"id","usando-faiss-para-una-bsqueda-eficiente-por-similaridad"),k(Ve,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),k(Ve,"href","#usando-faiss-para-una-bsqueda-eficiente-por-similaridad"),k(Me,"class","relative group"),k(ks,"href","https://faiss.ai/"),k(ks,"rel","nofollow")},m(e,i){s(document.head,m),d(e,b,i),x(f,e,i),d(e,j,i),d(e,h,i),s(h,v),s(v,R),x(C,R,null),s(h,I),s(h,w),s(w,$),d(e,D,i),Ps[P].m(e,i),d(e,H,i),d(e,F,i),s(F,B),s(F,A),s(A,L),s(F,W),d(e,G,i),x(T,e,i),d(e,Y,i),d(e,p,i),s(p,S),s(S,N),x(J,N,null),s(p,le),s(p,he),s(he,Zo),d(e,St,i),d(e,Q,i),s(Q,en),s(Q,Hs),s(Hs,sn),s(Q,an),s(Q,ta),s(ta,tn),s(Q,on),s(Q,oa),s(oa,nn),s(Q,rn),s(Q,na),s(na,ln),s(Q,dn),s(Q,ra),s(ra,un),s(Q,cn),d(e,Ot,i),d(e,Re,i),s(Re,mn),s(Re,la),s(la,pn),s(Re,fn),d(e,It,i),d(e,Se,i),s(Se,ss),s(Se,hn),s(Se,as),d(e,Mt,i),d(e,Oe,i),s(Oe,He),s(He,ia),x(ts,ia,null),s(Oe,_n),s(Oe,da),s(da,gn),d(e,Rt,i),d(e,Ls,i),s(Ls,bn),d(e,Ht,i),x(os,e,i),d(e,Lt,i),d(e,xe,i),s(xe,vn),s(xe,ua),s(ua,$n),s(xe,qn),s(xe,Fs),s(Fs,En),s(xe,xn),d(e,Ft,i),x(ns,e,i),d(e,Nt,i),x(rs,e,i),d(e,zt,i),d(e,X,i),s(X,yn),s(X,ca),s(ca,wn),s(X,kn),s(X,ma),s(ma,jn),s(X,Dn),s(X,pa),s(pa,Tn),s(X,Cn),s(X,fa),s(fa,An),s(X,Pn),s(X,ha),s(ha,Sn),s(X,On),d(e,Ut,i),x(ls,e,i),d(e,Gt,i),x(is,e,i),d(e,Bt,i),d(e,K,i),s(K,In),s(K,_a),s(_a,Mn),s(K,Rn),s(K,ga),s(ga,Hn),s(K,Ln),s(K,ba),s(ba,Fn),s(K,Nn),s(K,va),s(va,zn),s(K,Un),s(K,$a),s($a,Gn),s(K,Bn),d(e,Yt,i),x(ds,e,i),d(e,Vt,i),x(us,e,i),d(e,Wt,i),d(e,Z,i),s(Z,Yn),s(Z,qa),s(qa,Vn),s(Z,Wn),s(Z,Ea),s(Ea,Jn),s(Z,Qn),s(Z,xa),s(xa,Xn),s(Z,Kn),s(Z,Le),s(Le,Zn),s(Le,ya),s(ya,er),s(Z,sr),s(Z,wa),s(wa,ar),s(Z,tr),d(e,Jt,i),x(cs,e,i),d(e,Qt,i),d(e,Fe,i),s(Fe,or),s(Fe,ka),s(ka,nr),s(Fe,rr),d(e,Xt,i),x(ms,e,i),d(e,Kt,i),x(ps,e,i),d(e,Zt,i),d(e,Ne,i),s(Ne,lr),s(Ne,ja),s(ja,ir),s(Ne,dr),d(e,eo,i),x(fs,e,i),d(e,so,i),d(e,ae,i),s(ae,Da),s(Da,ee),s(ee,ao),s(ee,ur),s(ee,Ta),s(Ta,cr),s(ee,mr),s(ee,Ca),s(Ca,pr),s(ee,fr),s(ee,Aa),s(Aa,hr),s(ee,_r),s(ee,Pa),s(Pa,gr),s(ae,br),s(ae,_e),s(_e,te),s(te,Sa),s(Sa,vr),s(te,$r),s(te,Oa),s(Oa,qr),s(te,Er),s(te,Ia),s(Ia,xr),s(te,yr),s(te,Ma),s(Ma,wr),s(te,kr),s(te,Ra),s(Ra,jr),s(_e,Dr),s(_e,oe),s(oe,Ha),s(Ha,Tr),s(oe,Cr),s(oe,La),s(La,Ar),s(oe,Pr),s(oe,Fa),s(Fa,Sr),s(oe,Or),s(oe,Na),s(Na,Ir),s(oe,Mr),s(oe,za),s(za,Rr),s(_e,Hr),s(_e,ne),s(ne,Ua),s(Ua,Lr),s(ne,Fr),s(ne,Ga),s(Ga,Nr),s(ne,zr),s(ne,Ba),s(Ba,Ur),s(ne,Gr),s(ne,Ya),s(Ya,Br),s(ne,Yr),s(ne,Va),s(Va,Vr),s(_e,Wr),s(_e,re),s(re,Wa),s(Wa,Jr),s(re,Qr),s(re,Ja),s(Ja,Xr),s(re,Kr),s(re,Qa),s(Qa,Zr),s(re,el),s(re,Xa),s(Xa,sl),s(re,al),s(re,Ka),s(Ka,tl),d(e,to,i),d(e,ie,i),s(ie,ol),s(ie,Za),s(Za,nl),s(ie,rl),s(ie,et),s(et,ll),s(ie,il),s(ie,st),s(st,dl),s(ie,ul),d(e,oo,i),x(hs,e,i),d(e,no,i),x(_s,e,i),d(e,ro,i),d(e,Ns,i),s(Ns,cl),d(e,lo,i),x(ze,e,i),d(e,io,i),d(e,Ue,i),s(Ue,ml),s(Ue,at),s(at,pl),s(Ue,fl),d(e,uo,i),x(gs,e,i),d(e,co,i),d(e,zs,i),s(zs,hl),d(e,mo,i),x(bs,e,i),d(e,po,i),x(vs,e,i),d(e,fo,i),d(e,ye,i),s(ye,_l),s(ye,tt),s(tt,gl),s(ye,bl),s(ye,ot),s(ot,vl),s(ye,$l),d(e,ho,i),x($s,e,i),d(e,_o,i),d(e,Ge,i),s(Ge,ql),s(Ge,nt),s(nt,El),s(Ge,xl),d(e,go,i),d(e,Ie,i),s(Ie,Be),s(Be,rt),x(qs,rt,null),s(Ie,yl),s(Ie,lt),s(lt,wl),d(e,bo,i),d(e,z,i),s(z,kl),s(z,Us),s(Us,jl),s(z,Dl),s(z,it),s(it,Tl),s(z,Cl),s(z,dt),s(dt,Al),s(z,Pl),s(z,ut),s(ut,Sl),s(z,Ol),s(z,ct),s(ct,Il),s(z,Ml),s(z,Es),s(Es,Rl),s(z,Hl),s(z,mt),s(mt,Ll),s(z,Fl),s(z,xs),s(xs,Nl),s(z,zl),s(z,pt),s(pt,Ul),s(z,Gl),d(e,vo,i),Ss[ge].m(e,i),d(e,Gs,i),d(e,de,i),s(de,Bl),s(de,ft),s(ft,Yl),s(de,Vl),s(de,ht),s(ht,Wl),s(de,Jl),s(de,_t),s(_t,Ql),s(de,Xl),d(e,$o,i),x(ys,e,i),d(e,qo,i),d(e,Bs,i),s(Bs,Kl),d(e,Eo,i),Os[ve].m(e,i),d(e,Ys,i),d(e,Ye,i),s(Ye,Zl),s(Ye,gt),s(gt,ei),s(Ye,si),d(e,xo,i),d(e,Me,i),s(Me,Ve),s(Ve,bt),x(ws,bt,null),s(Me,ai),s(Me,vt),s(vt,ti),d(e,yo,i),d(e,se,i),s(se,oi),s(se,$t),s($t,ni),s(se,ri),s(se,ks),s(ks,li),s(se,ii),s(se,qt),s(qt,di),s(se,ui),s(se,Et),s(Et,ci),s(se,mi),d(e,wo,i),d(e,ue,i),s(ue,pi),s(ue,xt),s(xt,fi),s(ue,hi),s(ue,yt),s(yt,_i),s(ue,gi),s(ue,wt),s(wt,bi),s(ue,vi),d(e,ko,i),x(js,e,i),d(e,jo,i),d(e,we,i),s(we,$i),s(we,kt),s(kt,qi),s(we,Ei),s(we,jt),s(jt,xi),s(we,yi),d(e,Do,i),Is[qe].m(e,i),d(e,Vs,i),d(e,We,i),s(We,wi),s(We,Dt),s(Dt,ki),s(We,ji),d(e,To,i),x(Ds,e,i),d(e,Co,i),d(e,ke,i),s(ke,Di),s(ke,Tt),s(Tt,Ti),s(ke,Ci),s(ke,Ct),s(Ct,Ai),s(ke,Pi),d(e,Ao,i),x(Ts,e,i),d(e,Po,i),d(e,Ws,i),s(Ws,Si),d(e,So,i),x(Cs,e,i),d(e,Oo,i),x(As,e,i),d(e,Io,i),d(e,Js,i),s(Js,Oi),d(e,Mo,i),x(Je,e,i),Ro=!0},p(e,[i]){const Ms={};i&1&&(Ms.fw=e[0]),f.$set(Ms);let Qs=P;P=Li(e),P!==Qs&&(Ko(),_(Ps[Qs],1,1,()=>{Ps[Qs]=null}),Xo(),M=Ps[P],M||(M=Ps[P]=Hi[P](e),M.c()),g(M,1),M.m(H.parentNode,H));const At={};i&2&&(At.$$scope={dirty:i,ctx:e}),ze.$set(At);let Xs=ge;ge=Ni(e),ge!==Xs&&(Ko(),_(Ss[Xs],1,1,()=>{Ss[Xs]=null}),Xo(),be=Ss[ge],be||(be=Ss[ge]=Fi[ge](e),be.c()),g(be,1),be.m(Gs.parentNode,Gs));let Qe=ve;ve=Ui(e),ve!==Qe&&(Ko(),_(Os[Qe],1,1,()=>{Os[Qe]=null}),Xo(),$e=Os[ve],$e||($e=Os[ve]=zi[ve](e),$e.c()),g($e,1),$e.m(Ys.parentNode,Ys));let Ks=qe;qe=Bi(e),qe!==Ks&&(Ko(),_(Is[Ks],1,1,()=>{Is[Ks]=null}),Xo(),Ee=Is[qe],Ee||(Ee=Is[qe]=Gi[qe](e),Ee.c()),g(Ee,1),Ee.m(Vs.parentNode,Vs));const Rs={};i&2&&(Rs.$$scope={dirty:i,ctx:e}),Je.$set(Rs)},i(e){Ro||(g(f.$$.fragment,e),g(C.$$.fragment,e),g(M),g(T.$$.fragment,e),g(J.$$.fragment,e),g(ts.$$.fragment,e),g(os.$$.fragment,e),g(ns.$$.fragment,e),g(rs.$$.fragment,e),g(ls.$$.fragment,e),g(is.$$.fragment,e),g(ds.$$.fragment,e),g(us.$$.fragment,e),g(cs.$$.fragment,e),g(ms.$$.fragment,e),g(ps.$$.fragment,e),g(fs.$$.fragment,e),g(hs.$$.fragment,e),g(_s.$$.fragment,e),g(ze.$$.fragment,e),g(gs.$$.fragment,e),g(bs.$$.fragment,e),g(vs.$$.fragment,e),g($s.$$.fragment,e),g(qs.$$.fragment,e),g(be),g(ys.$$.fragment,e),g($e),g(ws.$$.fragment,e),g(js.$$.fragment,e),g(Ee),g(Ds.$$.fragment,e),g(Ts.$$.fragment,e),g(Cs.$$.fragment,e),g(As.$$.fragment,e),g(Je.$$.fragment,e),Ro=!0)},o(e){_(f.$$.fragment,e),_(C.$$.fragment,e),_(M),_(T.$$.fragment,e),_(J.$$.fragment,e),_(ts.$$.fragment,e),_(os.$$.fragment,e),_(ns.$$.fragment,e),_(rs.$$.fragment,e),_(ls.$$.fragment,e),_(is.$$.fragment,e),_(ds.$$.fragment,e),_(us.$$.fragment,e),_(cs.$$.fragment,e),_(ms.$$.fragment,e),_(ps.$$.fragment,e),_(fs.$$.fragment,e),_(hs.$$.fragment,e),_(_s.$$.fragment,e),_(ze.$$.fragment,e),_(gs.$$.fragment,e),_(bs.$$.fragment,e),_(vs.$$.fragment,e),_($s.$$.fragment,e),_(qs.$$.fragment,e),_(be),_(ys.$$.fragment,e),_($e),_(ws.$$.fragment,e),_(js.$$.fragment,e),_(Ee),_(Ds.$$.fragment,e),_(Ts.$$.fragment,e),_(Cs.$$.fragment,e),_(As.$$.fragment,e),_(Je.$$.fragment,e),Ro=!1},d(e){a(m),e&&a(b),y(f,e),e&&a(j),e&&a(h),y(C),e&&a(D),Ps[P].d(e),e&&a(H),e&&a(F),e&&a(G),y(T,e),e&&a(Y),e&&a(p),y(J),e&&a(St),e&&a(Q),e&&a(Ot),e&&a(Re),e&&a(It),e&&a(Se),e&&a(Mt),e&&a(Oe),y(ts),e&&a(Rt),e&&a(Ls),e&&a(Ht),y(os,e),e&&a(Lt),e&&a(xe),e&&a(Ft),y(ns,e),e&&a(Nt),y(rs,e),e&&a(zt),e&&a(X),e&&a(Ut),y(ls,e),e&&a(Gt),y(is,e),e&&a(Bt),e&&a(K),e&&a(Yt),y(ds,e),e&&a(Vt),y(us,e),e&&a(Wt),e&&a(Z),e&&a(Jt),y(cs,e),e&&a(Qt),e&&a(Fe),e&&a(Xt),y(ms,e),e&&a(Kt),y(ps,e),e&&a(Zt),e&&a(Ne),e&&a(eo),y(fs,e),e&&a(so),e&&a(ae),e&&a(to),e&&a(ie),e&&a(oo),y(hs,e),e&&a(no),y(_s,e),e&&a(ro),e&&a(Ns),e&&a(lo),y(ze,e),e&&a(io),e&&a(Ue),e&&a(uo),y(gs,e),e&&a(co),e&&a(zs),e&&a(mo),y(bs,e),e&&a(po),y(vs,e),e&&a(fo),e&&a(ye),e&&a(ho),y($s,e),e&&a(_o),e&&a(Ge),e&&a(go),e&&a(Ie),y(qs),e&&a(bo),e&&a(z),e&&a(vo),Ss[ge].d(e),e&&a(Gs),e&&a(de),e&&a($o),y(ys,e),e&&a(qo),e&&a(Bs),e&&a(Eo),Os[ve].d(e),e&&a(Ys),e&&a(Ye),e&&a(xo),e&&a(Me),y(ws),e&&a(yo),e&&a(se),e&&a(wo),e&&a(ue),e&&a(ko),y(js,e),e&&a(jo),e&&a(we),e&&a(Do),Is[qe].d(e),e&&a(Vs),e&&a(We),e&&a(To),y(Ds,e),e&&a(Co),e&&a(ke),e&&a(Ao),y(Ts,e),e&&a(Po),e&&a(Ws),e&&a(So),y(Cs,e),e&&a(Oo),y(As,e),e&&a(Io),e&&a(Js),e&&a(Mo),y(Je,e)}}}const nc={local:"bsqueda-semntica-con-faiss",sections:[{local:"usando-embeddings-para-la-bsqueda-semntica",title:"Usando _embeddings_ para la b\xFAsqueda sem\xE1ntica"},{local:"cargando-y-preparando-el-dataset",title:"Cargando y preparando el dataset"},{local:"creando-embeddings-de-texto",title:"Creando _embeddings_ de texto"},{local:"usando-faiss-para-una-bsqueda-eficiente-por-similaridad",title:"Usando FAISS para una b\xFAsqueda eficiente por similaridad"}],title:"B\xFAsqueda sem\xE1ntica con FAISS"};function rc(U,m,b){let f="pt";return Bu(()=>{const j=new URLSearchParams(window.location.search);b(0,f=j.get("fw")||"pt")}),[f]}class fc extends Nu{constructor(m){super();zu(this,m,rc,oc,Uu,{})}}export{fc as default,nc as metadata};
