import{S as sd,i as nd,s as ad,e as o,k as d,w as E,t as n,M as od,c as r,d as s,m as p,x,a as l,h as a,b as y,N as Zc,f as to,G as t,g as u,y as k,o as v,p as so,q as b,B as w,v as rd,n as no}from"../../chunks/vendor-hf-doc-builder.js";import{T as ed}from"../../chunks/Tip-hf-doc-builder.js";import{Y as ld}from"../../chunks/Youtube-hf-doc-builder.js";import{I as Fn}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as A}from"../../chunks/CodeBlock-hf-doc-builder.js";import{C as td}from"../../chunks/CourseFloatingBanner-hf-doc-builder.js";import{F as id}from"../../chunks/FrameworkSwitchCourse-hf-doc-builder.js";function ud(G){let m,q;return m=new td({props:{chapter:5,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"English",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter5/section6_tf.ipynb"},{label:"Fran\xE7ais",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/fr/chapter5/section6_tf.ipynb"},{label:"English",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter5/section6_tf.ipynb"},{label:"Fran\xE7ais",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/fr/chapter5/section6_tf.ipynb"}]}}),{c(){E(m.$$.fragment)},l(f){x(m.$$.fragment,f)},m(f,j){k(m,f,j),q=!0},i(f){q||(b(m.$$.fragment,f),q=!0)},o(f){v(m.$$.fragment,f),q=!1},d(f){w(m,f)}}}function cd(G){let m,q;return m=new td({props:{chapter:5,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"English",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter5/section6_pt.ipynb"},{label:"Fran\xE7ais",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/fr/chapter5/section6_pt.ipynb"},{label:"English",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter5/section6_pt.ipynb"},{label:"Fran\xE7ais",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/fr/chapter5/section6_pt.ipynb"}]}}),{c(){E(m.$$.fragment)},l(f){x(m.$$.fragment,f)},m(f,j){k(m,f,j),q=!0},i(f){q||(b(m.$$.fragment,f),q=!0)},o(f){v(m.$$.fragment,f),q=!1},d(f){w(m,f)}}}function dd(G){let m,q,f,j,_,$,S,g,C,D,z,O,T,I,N,R,Y,P,Q,B,U,L,V;return{c(){m=o("p"),q=n("\u270F\uFE0F "),f=o("strong"),j=n("Essayez !"),_=n(" Voyez si vous pouvez utiliser "),$=o("code"),S=n("Dataset.map()"),g=n(" pour exploser la colonne "),C=o("code"),D=n("comments"),z=n(" de "),O=o("code"),T=n("issues_dataset"),I=d(),N=o("em"),R=n("sans"),Y=n(" recourir \xE0 l\u2019utilisation de Pandas. C\u2019est un peu d\xE9licat. La section "),P=o("a"),Q=n("\xAB Batch mapping \xBB"),B=n(" de la documentation \u{1F917} "),U=o("em"),L=n("Datasets"),V=n(" peut \xEAtre utile pour cette t\xE2che."),this.h()},l(c){m=r(c,"P",{});var h=l(m);q=a(h,"\u270F\uFE0F "),f=r(h,"STRONG",{});var F=l(f);j=a(F,"Essayez !"),F.forEach(s),_=a(h," Voyez si vous pouvez utiliser "),$=r(h,"CODE",{});var W=l($);S=a(W,"Dataset.map()"),W.forEach(s),g=a(h," pour exploser la colonne "),C=r(h,"CODE",{});var J=l(C);D=a(J,"comments"),J.forEach(s),z=a(h," de "),O=r(h,"CODE",{});var K=l(O);T=a(K,"issues_dataset"),K.forEach(s),I=p(h),N=r(h,"EM",{});var re=l(N);R=a(re,"sans"),re.forEach(s),Y=a(h," recourir \xE0 l\u2019utilisation de Pandas. C\u2019est un peu d\xE9licat. La section "),P=r(h,"A",{href:!0,rel:!0});var Pe=l(P);Q=a(Pe,"\xAB Batch mapping \xBB"),Pe.forEach(s),B=a(h," de la documentation \u{1F917} "),U=r(h,"EM",{});var He=l(U);L=a(He,"Datasets"),He.forEach(s),V=a(h," peut \xEAtre utile pour cette t\xE2che."),h.forEach(s),this.h()},h(){y(P,"href","https://huggingface.co/docs/datasets/v1.12.1/about_map_batch.html?batch-mapping#batch-mapping"),y(P,"rel","nofollow")},m(c,h){u(c,m,h),t(m,q),t(m,f),t(f,j),t(m,_),t(m,$),t($,S),t(m,g),t(m,C),t(C,D),t(m,z),t(m,O),t(O,T),t(m,I),t(m,N),t(N,R),t(m,Y),t(m,P),t(P,Q),t(m,B),t(m,U),t(U,L),t(m,V)},d(c){c&&s(m)}}}function pd(G){let m,q,f,j,_,$,S,g,C,D,z,O,T,I,N,R,Y,P,Q,B,U,L,V;return m=new A({props:{code:`from transformers import AutoTokenizer, TFAutoModel

model_ckpt = "sentence-transformers/multi-qa-mpnet-base-dot-v1"
tokenizer = AutoTokenizer.from_pretrained(model_ckpt)
model = TFAutoModel.from_pretrained(model_ckpt, from_pt=True)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, TFAutoModel

model_ckpt = <span class="hljs-string">&quot;sentence-transformers/multi-qa-mpnet-base-dot-v1&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_ckpt)
model = TFAutoModel.from_pretrained(model_ckpt, from_pt=<span class="hljs-literal">True</span>)`}}),{c(){E(m.$$.fragment),q=d(),f=o("p"),j=n("Notez que nous avons d\xE9fini "),_=o("code"),$=n("from_pt=True"),S=n(" comme argument de la m\xE9thode "),g=o("code"),C=n("from_pretrained()"),D=n(". C\u2019est parce que le point de contr\xF4le "),z=o("code"),O=n("multi-qa-mpnet-base-dot-v1"),T=n(" n\u2019a que des poids PyTorch. Donc d\xE9finir "),I=o("code"),N=n("from_pt=True"),R=n(" converti automatiquement au format TensorFlow pour nous. Comme vous pouvez le voir, il est tr\xE8s simple de passer d\u2019un "),Y=o("em"),P=n("framework"),Q=n(" \xE0 l\u2019autre dans \u{1F917} "),B=o("em"),U=n("Transformers"),L=n(" !")},l(c){x(m.$$.fragment,c),q=p(c),f=r(c,"P",{});var h=l(f);j=a(h,"Notez que nous avons d\xE9fini "),_=r(h,"CODE",{});var F=l(_);$=a(F,"from_pt=True"),F.forEach(s),S=a(h," comme argument de la m\xE9thode "),g=r(h,"CODE",{});var W=l(g);C=a(W,"from_pretrained()"),W.forEach(s),D=a(h,". C\u2019est parce que le point de contr\xF4le "),z=r(h,"CODE",{});var J=l(z);O=a(J,"multi-qa-mpnet-base-dot-v1"),J.forEach(s),T=a(h," n\u2019a que des poids PyTorch. Donc d\xE9finir "),I=r(h,"CODE",{});var K=l(I);N=a(K,"from_pt=True"),K.forEach(s),R=a(h," converti automatiquement au format TensorFlow pour nous. Comme vous pouvez le voir, il est tr\xE8s simple de passer d\u2019un "),Y=r(h,"EM",{});var re=l(Y);P=a(re,"framework"),re.forEach(s),Q=a(h," \xE0 l\u2019autre dans \u{1F917} "),B=r(h,"EM",{});var Pe=l(B);U=a(Pe,"Transformers"),Pe.forEach(s),L=a(h," !"),h.forEach(s)},m(c,h){k(m,c,h),u(c,q,h),u(c,f,h),t(f,j),t(f,_),t(_,$),t(f,S),t(f,g),t(g,C),t(f,D),t(f,z),t(z,O),t(f,T),t(f,I),t(I,N),t(f,R),t(f,Y),t(Y,P),t(f,Q),t(f,B),t(B,U),t(f,L),V=!0},i(c){V||(b(m.$$.fragment,c),V=!0)},o(c){v(m.$$.fragment,c),V=!1},d(c){w(m,c),c&&s(q),c&&s(f)}}}function md(G){let m,q,f,j,_,$,S;return m=new A({props:{code:`from transformers import AutoTokenizer, AutoModel

model_ckpt = "sentence-transformers/multi-qa-mpnet-base-dot-v1"
tokenizer = AutoTokenizer.from_pretrained(model_ckpt)
model = AutoModel.from_pretrained(model_ckpt)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModel

model_ckpt = <span class="hljs-string">&quot;sentence-transformers/multi-qa-mpnet-base-dot-v1&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_ckpt)
model = AutoModel.from_pretrained(model_ckpt)`}}),$=new A({props:{code:`import torch

device = torch.device("cuda")
model.to(device)`,highlighted:`<span class="hljs-keyword">import</span> torch

device = torch.device(<span class="hljs-string">&quot;cuda&quot;</span>)
model.to(device)`}}),{c(){E(m.$$.fragment),q=d(),f=o("p"),j=n("Pour acc\xE9l\xE9rer le processus, il est utile de placer le mod\xE8le et les entr\xE9es sur un p\xE9riph\xE9rique GPU, alors faisons-le maintenant :"),_=d(),E($.$$.fragment)},l(g){x(m.$$.fragment,g),q=p(g),f=r(g,"P",{});var C=l(f);j=a(C,"Pour acc\xE9l\xE9rer le processus, il est utile de placer le mod\xE8le et les entr\xE9es sur un p\xE9riph\xE9rique GPU, alors faisons-le maintenant :"),C.forEach(s),_=p(g),x($.$$.fragment,g)},m(g,C){k(m,g,C),u(g,q,C),u(g,f,C),t(f,j),u(g,_,C),k($,g,C),S=!0},i(g){S||(b(m.$$.fragment,g),b($.$$.fragment,g),S=!0)},o(g){v(m.$$.fragment,g),v($.$$.fragment,g),S=!1},d(g){w(m,g),g&&s(q),g&&s(f),g&&s(_),w($,g)}}}function fd(G){let m,q,f,j,_,$,S,g,C,D,z,O,T,I,N,R,Y,P,Q,B,U,L,V;return m=new A({props:{code:`def get_embeddings(text_list):
    encoded_input = tokenizer(
        text_list, padding=True, truncation=True, return_tensors="tf"
    )
    encoded_input = {k: v for k, v in encoded_input.items()}
    model_output = model(**encoded_input)
    return cls_pooling(model_output)`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_embeddings</span>(<span class="hljs-params">text_list</span>):
    encoded_input = tokenizer(
        text_list, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>
    )
    encoded_input = {k: v <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> encoded_input.items()}
    model_output = model(**encoded_input)
    <span class="hljs-keyword">return</span> cls_pooling(model_output)`}}),$=new A({props:{code:`embedding = get_embeddings(comments_dataset["text"][0])
embedding.shape`,highlighted:`embedding = get_embeddings(comments_dataset[<span class="hljs-string">&quot;text&quot;</span>][<span class="hljs-number">0</span>])
embedding.shape`}}),g=new A({props:{code:"TensorShape([1, 768])",highlighted:'TensorShape([<span class="hljs-number">1</span>, <span class="hljs-number">768</span>])'}}),L=new A({props:{code:`embeddings_dataset = comments_dataset.map(
    lambda x: {"embeddings": get_embeddings(x["text"]).numpy()[0]}
)`,highlighted:`embeddings_dataset = comments_dataset.<span class="hljs-built_in">map</span>(
    <span class="hljs-keyword">lambda</span> x: {<span class="hljs-string">&quot;embeddings&quot;</span>: get_embeddings(x[<span class="hljs-string">&quot;text&quot;</span>]).numpy()[<span class="hljs-number">0</span>]}
)`}}),{c(){E(m.$$.fragment),q=d(),f=o("p"),j=n("Nous pouvons tester le fonctionnement de la fonction en lui donnant la premi\xE8re entr\xE9e textuelle de notre corpus et en inspectant la forme de sortie :"),_=d(),E($.$$.fragment),S=d(),E(g.$$.fragment),C=d(),D=o("p"),z=n("Super ! Nous avons converti la premi\xE8re entr\xE9e de notre corpus en un vecteur \xE0 768 dimensions. Nous pouvons utiliser "),O=o("code"),T=n("Dataset.map()"),I=n(" pour appliquer notre fonction "),N=o("code"),R=n("get_embeddings()"),Y=n(" \xE0 chaque ligne de notre corpus. Cr\xE9ons donc une nouvelle colonne "),P=o("code"),Q=n("embeddings"),B=n(" comme suit :"),U=d(),E(L.$$.fragment)},l(c){x(m.$$.fragment,c),q=p(c),f=r(c,"P",{});var h=l(f);j=a(h,"Nous pouvons tester le fonctionnement de la fonction en lui donnant la premi\xE8re entr\xE9e textuelle de notre corpus et en inspectant la forme de sortie :"),h.forEach(s),_=p(c),x($.$$.fragment,c),S=p(c),x(g.$$.fragment,c),C=p(c),D=r(c,"P",{});var F=l(D);z=a(F,"Super ! Nous avons converti la premi\xE8re entr\xE9e de notre corpus en un vecteur \xE0 768 dimensions. Nous pouvons utiliser "),O=r(F,"CODE",{});var W=l(O);T=a(W,"Dataset.map()"),W.forEach(s),I=a(F," pour appliquer notre fonction "),N=r(F,"CODE",{});var J=l(N);R=a(J,"get_embeddings()"),J.forEach(s),Y=a(F," \xE0 chaque ligne de notre corpus. Cr\xE9ons donc une nouvelle colonne "),P=r(F,"CODE",{});var K=l(P);Q=a(K,"embeddings"),K.forEach(s),B=a(F," comme suit :"),F.forEach(s),U=p(c),x(L.$$.fragment,c)},m(c,h){k(m,c,h),u(c,q,h),u(c,f,h),t(f,j),u(c,_,h),k($,c,h),u(c,S,h),k(g,c,h),u(c,C,h),u(c,D,h),t(D,z),t(D,O),t(O,T),t(D,I),t(D,N),t(N,R),t(D,Y),t(D,P),t(P,Q),t(D,B),u(c,U,h),k(L,c,h),V=!0},i(c){V||(b(m.$$.fragment,c),b($.$$.fragment,c),b(g.$$.fragment,c),b(L.$$.fragment,c),V=!0)},o(c){v(m.$$.fragment,c),v($.$$.fragment,c),v(g.$$.fragment,c),v(L.$$.fragment,c),V=!1},d(c){w(m,c),c&&s(q),c&&s(f),c&&s(_),w($,c),c&&s(S),w(g,c),c&&s(C),c&&s(D),c&&s(U),w(L,c)}}}function hd(G){let m,q,f,j,_,$,S,g,C,D,z,O,T,I,N,R,Y,P,Q,B,U,L,V;return m=new A({props:{code:`def get_embeddings(text_list):
    encoded_input = tokenizer(
        text_list, padding=True, truncation=True, return_tensors="pt"
    )
    encoded_input = {k: v.to(device) for k, v in encoded_input.items()}
    model_output = model(**encoded_input)
    return cls_pooling(model_output)`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_embeddings</span>(<span class="hljs-params">text_list</span>):
    encoded_input = tokenizer(
        text_list, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>
    )
    encoded_input = {k: v.to(device) <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> encoded_input.items()}
    model_output = model(**encoded_input)
    <span class="hljs-keyword">return</span> cls_pooling(model_output)`}}),$=new A({props:{code:`embedding = get_embeddings(comments_dataset["text"][0])
embedding.shape`,highlighted:`embedding = get_embeddings(comments_dataset[<span class="hljs-string">&quot;text&quot;</span>][<span class="hljs-number">0</span>])
embedding.shape`}}),g=new A({props:{code:"torch.Size([1, 768])",highlighted:'torch.Size([<span class="hljs-number">1</span>, <span class="hljs-number">768</span>])'}}),L=new A({props:{code:`embeddings_dataset = comments_dataset.map(
    lambda x: {"embeddings": get_embeddings(x["text"]).detach().cpu().numpy()[0]}
)`,highlighted:`embeddings_dataset = comments_dataset.<span class="hljs-built_in">map</span>(
    <span class="hljs-keyword">lambda</span> x: {<span class="hljs-string">&quot;embeddings&quot;</span>: get_embeddings(x[<span class="hljs-string">&quot;text&quot;</span>]).detach().cpu().numpy()[<span class="hljs-number">0</span>]}
)`}}),{c(){E(m.$$.fragment),q=d(),f=o("p"),j=n("Nous pouvons tester le fonctionnement de la fonction en lui donnant la premi\xE8re entr\xE9e textuelle de notre corpus et en inspectant la forme de sortie :"),_=d(),E($.$$.fragment),S=d(),E(g.$$.fragment),C=d(),D=o("p"),z=n("Super ! Nous avons converti la premi\xE8re entr\xE9e de notre corpus en un vecteur \xE0 768 dimensions. Nous pouvons utiliser "),O=o("code"),T=n("Dataset.map()"),I=n(" pour appliquer notre fonction "),N=o("code"),R=n("get_embeddings()"),Y=n(" \xE0 chaque ligne de notre corpus. Cr\xE9ons donc une nouvelle colonne "),P=o("code"),Q=n("embeddings"),B=n(" comme suit :"),U=d(),E(L.$$.fragment)},l(c){x(m.$$.fragment,c),q=p(c),f=r(c,"P",{});var h=l(f);j=a(h,"Nous pouvons tester le fonctionnement de la fonction en lui donnant la premi\xE8re entr\xE9e textuelle de notre corpus et en inspectant la forme de sortie :"),h.forEach(s),_=p(c),x($.$$.fragment,c),S=p(c),x(g.$$.fragment,c),C=p(c),D=r(c,"P",{});var F=l(D);z=a(F,"Super ! Nous avons converti la premi\xE8re entr\xE9e de notre corpus en un vecteur \xE0 768 dimensions. Nous pouvons utiliser "),O=r(F,"CODE",{});var W=l(O);T=a(W,"Dataset.map()"),W.forEach(s),I=a(F," pour appliquer notre fonction "),N=r(F,"CODE",{});var J=l(N);R=a(J,"get_embeddings()"),J.forEach(s),Y=a(F," \xE0 chaque ligne de notre corpus. Cr\xE9ons donc une nouvelle colonne "),P=r(F,"CODE",{});var K=l(P);Q=a(K,"embeddings"),K.forEach(s),B=a(F," comme suit :"),F.forEach(s),U=p(c),x(L.$$.fragment,c)},m(c,h){k(m,c,h),u(c,q,h),u(c,f,h),t(f,j),u(c,_,h),k($,c,h),u(c,S,h),k(g,c,h),u(c,C,h),u(c,D,h),t(D,z),t(D,O),t(O,T),t(D,I),t(D,N),t(N,R),t(D,Y),t(D,P),t(P,Q),t(D,B),u(c,U,h),k(L,c,h),V=!0},i(c){V||(b(m.$$.fragment,c),b($.$$.fragment,c),b(g.$$.fragment,c),b(L.$$.fragment,c),V=!0)},o(c){v(m.$$.fragment,c),v($.$$.fragment,c),v(g.$$.fragment,c),v(L.$$.fragment,c),V=!1},d(c){w(m,c),c&&s(q),c&&s(f),c&&s(_),w($,c),c&&s(S),w(g,c),c&&s(C),c&&s(D),c&&s(U),w(L,c)}}}function _d(G){let m,q,f,j;return m=new A({props:{code:`question = "How can I load a dataset offline?"
question_embedding = get_embeddings([question]).numpy()
question_embedding.shape`,highlighted:`question = <span class="hljs-string">&quot;How can I load a dataset offline?&quot;</span>
question_embedding = get_embeddings([question]).numpy()
question_embedding.shape`}}),f=new A({props:{code:"(1, 768)",highlighted:'(<span class="hljs-number">1</span>, <span class="hljs-number">768</span>)'}}),{c(){E(m.$$.fragment),q=d(),E(f.$$.fragment)},l(_){x(m.$$.fragment,_),q=p(_),x(f.$$.fragment,_)},m(_,$){k(m,_,$),u(_,q,$),k(f,_,$),j=!0},i(_){j||(b(m.$$.fragment,_),b(f.$$.fragment,_),j=!0)},o(_){v(m.$$.fragment,_),v(f.$$.fragment,_),j=!1},d(_){w(m,_),_&&s(q),w(f,_)}}}function gd(G){let m,q,f,j;return m=new A({props:{code:`question = "How can I load a dataset offline?"
question_embedding = get_embeddings([question]).cpu().detach().numpy()
question_embedding.shape`,highlighted:`question = <span class="hljs-string">&quot;How can I load a dataset offline?&quot;</span>
question_embedding = get_embeddings([question]).cpu().detach().numpy()
question_embedding.shape`}}),f=new A({props:{code:"torch.Size([1, 768])",highlighted:'torch.Size([<span class="hljs-number">1</span>, <span class="hljs-number">768</span>])'}}),{c(){E(m.$$.fragment),q=d(),E(f.$$.fragment)},l(_){x(m.$$.fragment,_),q=p(_),x(f.$$.fragment,_)},m(_,$){k(m,_,$),u(_,q,$),k(f,_,$),j=!0},i(_){j||(b(m.$$.fragment,_),b(f.$$.fragment,_),j=!0)},o(_){v(m.$$.fragment,_),v(f.$$.fragment,_),j=!1},d(_){w(m,_),_&&s(q),w(f,_)}}}function vd(G){let m,q,f,j,_,$,S,g,C,D,z;return{c(){m=o("p"),q=n("\u270F\uFE0F "),f=o("strong"),j=n("Essayez !"),_=n(" Cr\xE9ez votre propre requ\xEAte et voyez si vous pouvez trouver une r\xE9ponse dans les documents r\xE9cup\xE9r\xE9s. Vous devrez peut-\xEAtre augmenter le param\xE8tre "),$=o("code"),S=n("k"),g=n(" dans "),C=o("code"),D=n("Dataset.get_nearest_examples()"),z=n(" pour \xE9largir la recherche.")},l(O){m=r(O,"P",{});var T=l(m);q=a(T,"\u270F\uFE0F "),f=r(T,"STRONG",{});var I=l(f);j=a(I,"Essayez !"),I.forEach(s),_=a(T," Cr\xE9ez votre propre requ\xEAte et voyez si vous pouvez trouver une r\xE9ponse dans les documents r\xE9cup\xE9r\xE9s. Vous devrez peut-\xEAtre augmenter le param\xE8tre "),$=r(T,"CODE",{});var N=l($);S=a(N,"k"),N.forEach(s),g=a(T," dans "),C=r(T,"CODE",{});var R=l(C);D=a(R,"Dataset.get_nearest_examples()"),R.forEach(s),z=a(T," pour \xE9largir la recherche."),T.forEach(s)},m(O,T){u(O,m,T),t(m,q),t(m,f),t(f,j),t(m,_),t(m,$),t($,S),t(m,g),t(m,C),t(C,D),t(m,z)},d(O){O&&s(m)}}}function bd(G){let m,q,f,j,_,$,S,g,C,D,z,O,T,I,N,R,Y,P,Q,B,U,L,V,c,h,F,W,J,K,re,Pe,He,ao,Ln,te,oo,Ft,ro,lo,os,io,uo,rs,co,po,ls,mo,fo,Hn,Lt,ho,zn,Me,st,Vi,_o,nt,Yi,Un,Re,ze,is,at,go,us,vo,Gn,ke,bo,cs,$o,qo,ds,Eo,xo,Vn,ot,Yn,we,ko,ps,wo,yo,Ht,jo,Do,Bn,rt,Wn,lt,Jn,X,Co,ms,To,Ao,fs,So,Oo,hs,Io,No,_s,Po,Mo,gs,Ro,Fo,vs,Lo,Ho,Qn,it,Xn,ut,Kn,Z,zo,bs,Uo,Go,$s,Vo,Yo,qs,Bo,Wo,Es,Jo,Qo,xs,Xo,Ko,Zn,ct,ea,dt,ta,ee,Zo,ks,er,tr,ws,sr,nr,ys,ar,or,pt,js,rr,lr,Ds,ir,ur,sa,mt,na,Ue,cr,Cs,dr,pr,aa,ft,oa,ht,ra,Ge,mr,Ts,fr,hr,la,_t,ia,le,As,se,ua,_r,Ss,gr,vr,Os,br,$r,Is,qr,Er,Ns,xr,kr,ge,ie,Ps,wr,yr,Ms,jr,Dr,Rs,Cr,Tr,Fs,Ar,Sr,Ls,Or,Ir,ue,Hs,Nr,Pr,zs,Mr,Rr,Us,Fr,Lr,Gs,Hr,zr,Vs,Ur,Gr,ce,Ys,Vr,Yr,Bs,Br,Wr,Ws,Jr,Qr,Js,Xr,Kr,Qs,Zr,el,de,Xs,tl,sl,Ks,nl,al,Zs,ol,rl,en,ll,il,tn,ul,ca,pe,cl,sn,dl,pl,nn,ml,fl,an,hl,_l,da,gt,pa,vt,ma,zt,gl,fa,Ve,ha,Ye,vl,on,bl,$l,_a,bt,ga,Ut,ql,va,$t,ba,qt,$a,ye,El,rn,xl,kl,ln,wl,yl,qa,Et,Ea,Gt,jl,xa,Fe,Be,un,xt,Dl,cn,Cl,ka,M,Tl,Vt,Al,Sl,dn,Ol,Il,pn,Nl,Pl,mn,Ml,Rl,fn,Fl,Ll,kt,Hl,zl,hn,Ul,Gl,wt,Vl,Yl,_n,Bl,Wl,gn,Jl,Ql,vn,Xl,Kl,bn,Zl,ei,wa,ve,be,Yt,ne,ti,$n,si,ni,qn,ai,oi,En,ri,li,xn,ii,ui,ya,yt,ja,Bt,ci,Da,$e,qe,Wt,We,di,kn,pi,mi,Ca,Le,Je,wn,jt,fi,yn,hi,Ta,ae,_i,jn,gi,vi,Dn,bi,$i,Dt,qi,Ei,Cn,xi,ki,Aa,me,wi,Tn,yi,ji,An,Di,Ci,Sn,Ti,Ai,Sa,Ct,Oa,Qe,Si,On,Oi,Ii,Ia,Ee,xe,Jt,Qt,Ni,Na,Tt,Pa,fe,Pi,In,Mi,Ri,Nn,Fi,Li,Pn,Hi,zi,Ma,At,Ra,Xt,Ui,Fa,St,La,Ot,Ha,Kt,Gi,za,Xe,Ua;f=new id({props:{fw:G[0]}}),g=new Fn({});const Bi=[cd,ud],It=[];function Wi(e,i){return e[0]==="pt"?0:1}T=Wi(G),I=It[T]=Bi[T](G),h=new ld({props:{id:"OATCgQtNX2o"}}),re=new Fn({}),at=new Fn({}),ot=new A({props:{code:`from huggingface_hub import hf_hub_url

data_files = hf_hub_url(
    repo_id="lewtun/github-issues",
    filename="datasets-issues-with-hf-doc-builder.jsonl",
    repo_type="dataset",
)`,highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> hf_hub_url

data_files = hf_hub_url(
    repo_id=<span class="hljs-string">&quot;lewtun/github-issues&quot;</span>,
    filename=<span class="hljs-string">&quot;datasets-issues-with-hf-doc-builder.jsonl&quot;</span>,
    repo_type=<span class="hljs-string">&quot;dataset&quot;</span>,
)`}}),rt=new A({props:{code:`from datasets import load_dataset

issues_dataset = load_dataset("json", data_files=data_files, split="train")
issues_dataset`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

issues_dataset = load_dataset(<span class="hljs-string">&quot;json&quot;</span>, data_files=data_files, split=<span class="hljs-string">&quot;train&quot;</span>)
issues_dataset`}}),lt=new A({props:{code:`Dataset({
    features: ['url', 'repository_url', 'labels_url', 'comments_url', 'events_url', 'html_url', 'id', 'node_id', 'number', 'title', 'user', 'labels', 'state', 'locked', 'assignee', 'assignees', 'milestone', 'comments', 'created_at', 'updated_at', 'closed_at', 'author_association', 'active_lock_reason', 'pull_request', 'body', 'performed_via_github_app', 'is_pull_request'],
    num_rows: 2855
})`,highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;url&#x27;</span>, <span class="hljs-string">&#x27;repository_url&#x27;</span>, <span class="hljs-string">&#x27;labels_url&#x27;</span>, <span class="hljs-string">&#x27;comments_url&#x27;</span>, <span class="hljs-string">&#x27;events_url&#x27;</span>, <span class="hljs-string">&#x27;html_url&#x27;</span>, <span class="hljs-string">&#x27;id&#x27;</span>, <span class="hljs-string">&#x27;node_id&#x27;</span>, <span class="hljs-string">&#x27;number&#x27;</span>, <span class="hljs-string">&#x27;title&#x27;</span>, <span class="hljs-string">&#x27;user&#x27;</span>, <span class="hljs-string">&#x27;labels&#x27;</span>, <span class="hljs-string">&#x27;state&#x27;</span>, <span class="hljs-string">&#x27;locked&#x27;</span>, <span class="hljs-string">&#x27;assignee&#x27;</span>, <span class="hljs-string">&#x27;assignees&#x27;</span>, <span class="hljs-string">&#x27;milestone&#x27;</span>, <span class="hljs-string">&#x27;comments&#x27;</span>, <span class="hljs-string">&#x27;created_at&#x27;</span>, <span class="hljs-string">&#x27;updated_at&#x27;</span>, <span class="hljs-string">&#x27;closed_at&#x27;</span>, <span class="hljs-string">&#x27;author_association&#x27;</span>, <span class="hljs-string">&#x27;active_lock_reason&#x27;</span>, <span class="hljs-string">&#x27;pull_request&#x27;</span>, <span class="hljs-string">&#x27;body&#x27;</span>, <span class="hljs-string">&#x27;performed_via_github_app&#x27;</span>, <span class="hljs-string">&#x27;is_pull_request&#x27;</span>],
    num_rows: <span class="hljs-number">2855</span>
})`}}),it=new A({props:{code:`issues_dataset = issues_dataset.filter(
    lambda x: (x["is_pull_request"] == False and len(x["comments"]) > 0)
)
issues_dataset`,highlighted:`issues_dataset = issues_dataset.<span class="hljs-built_in">filter</span>(
    <span class="hljs-keyword">lambda</span> x: (x[<span class="hljs-string">&quot;is_pull_request&quot;</span>] == <span class="hljs-literal">False</span> <span class="hljs-keyword">and</span> <span class="hljs-built_in">len</span>(x[<span class="hljs-string">&quot;comments&quot;</span>]) &gt; <span class="hljs-number">0</span>)
)
issues_dataset`}}),ut=new A({props:{code:`Dataset({
    features: ['url', 'repository_url', 'labels_url', 'comments_url', 'events_url', 'html_url', 'id', 'node_id', 'number', 'title', 'user', 'labels', 'state', 'locked', 'assignee', 'assignees', 'milestone', 'comments', 'created_at', 'updated_at', 'closed_at', 'author_association', 'active_lock_reason', 'pull_request', 'body', 'performed_via_github_app', 'is_pull_request'],
    num_rows: 771
})`,highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;url&#x27;</span>, <span class="hljs-string">&#x27;repository_url&#x27;</span>, <span class="hljs-string">&#x27;labels_url&#x27;</span>, <span class="hljs-string">&#x27;comments_url&#x27;</span>, <span class="hljs-string">&#x27;events_url&#x27;</span>, <span class="hljs-string">&#x27;html_url&#x27;</span>, <span class="hljs-string">&#x27;id&#x27;</span>, <span class="hljs-string">&#x27;node_id&#x27;</span>, <span class="hljs-string">&#x27;number&#x27;</span>, <span class="hljs-string">&#x27;title&#x27;</span>, <span class="hljs-string">&#x27;user&#x27;</span>, <span class="hljs-string">&#x27;labels&#x27;</span>, <span class="hljs-string">&#x27;state&#x27;</span>, <span class="hljs-string">&#x27;locked&#x27;</span>, <span class="hljs-string">&#x27;assignee&#x27;</span>, <span class="hljs-string">&#x27;assignees&#x27;</span>, <span class="hljs-string">&#x27;milestone&#x27;</span>, <span class="hljs-string">&#x27;comments&#x27;</span>, <span class="hljs-string">&#x27;created_at&#x27;</span>, <span class="hljs-string">&#x27;updated_at&#x27;</span>, <span class="hljs-string">&#x27;closed_at&#x27;</span>, <span class="hljs-string">&#x27;author_association&#x27;</span>, <span class="hljs-string">&#x27;active_lock_reason&#x27;</span>, <span class="hljs-string">&#x27;pull_request&#x27;</span>, <span class="hljs-string">&#x27;body&#x27;</span>, <span class="hljs-string">&#x27;performed_via_github_app&#x27;</span>, <span class="hljs-string">&#x27;is_pull_request&#x27;</span>],
    num_rows: <span class="hljs-number">771</span>
})`}}),ct=new A({props:{code:`columns = issues_dataset.column_names
columns_to_keep = ["title", "body", "html_url", "comments"]
columns_to_remove = set(columns_to_keep).symmetric_difference(columns)
issues_dataset = issues_dataset.remove_columns(columns_to_remove)
issues_dataset`,highlighted:`columns = issues_dataset.column_names
columns_to_keep = [<span class="hljs-string">&quot;title&quot;</span>, <span class="hljs-string">&quot;body&quot;</span>, <span class="hljs-string">&quot;html_url&quot;</span>, <span class="hljs-string">&quot;comments&quot;</span>]
columns_to_remove = <span class="hljs-built_in">set</span>(columns_to_keep).symmetric_difference(columns)
issues_dataset = issues_dataset.remove_columns(columns_to_remove)
issues_dataset`}}),dt=new A({props:{code:`Dataset({
    features: ['html_url', 'title', 'comments', 'body'],
    num_rows: 771
})`,highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;html_url&#x27;</span>, <span class="hljs-string">&#x27;title&#x27;</span>, <span class="hljs-string">&#x27;comments&#x27;</span>, <span class="hljs-string">&#x27;body&#x27;</span>],
    num_rows: <span class="hljs-number">771</span>
})`}}),mt=new A({props:{code:`issues_dataset.set_format("pandas")
df = issues_dataset[:]`,highlighted:`issues_dataset.set_format(<span class="hljs-string">&quot;pandas&quot;</span>)
df = issues_dataset[:]`}}),ft=new A({props:{code:'df["comments"][0].tolist()',highlighted:'df[<span class="hljs-string">&quot;comments&quot;</span>][<span class="hljs-number">0</span>].tolist()'}}),ht=new A({props:{code:`['the bug code locate in \uFF1A\\r\\n    if data_args.task_name is not None:\\r\\n        # Downloading and loading a dataset from the hub.\\r\\n        datasets = load_dataset("glue", data_args.task_name, cache_dir=model_args.cache_dir)',
 'Hi @jinec,\\r\\n\\r\\nFrom time to time we get this kind of \`ConnectionError\` coming from the github.com website: https://raw.githubusercontent.com\\r\\n\\r\\nNormally, it should work if you wait a little and then retry.\\r\\n\\r\\nCould you please confirm if the problem persists?',
 'cannot connect\uFF0Ceven by Web browser\uFF0Cplease check that  there is some  problems\u3002',
 'I can access https://raw.githubusercontent.com/huggingface/datasets/1.7.0/datasets/glue/glue.py without problem...']`,highlighted:`[<span class="hljs-string">&#x27;the bug code locate in \uFF1A\\r\\n    if data_args.task_name is not None:\\r\\n        # Downloading and loading a dataset from the hub.\\r\\n        datasets = load_dataset(&quot;glue&quot;, data_args.task_name, cache_dir=model_args.cache_dir)&#x27;</span>,
 <span class="hljs-string">&#x27;Hi @jinec,\\r\\n\\r\\nFrom time to time we get this kind of \`ConnectionError\` coming from the github.com website: https://raw.githubusercontent.com\\r\\n\\r\\nNormally, it should work if you wait a little and then retry.\\r\\n\\r\\nCould you please confirm if the problem persists?&#x27;</span>,
 <span class="hljs-string">&#x27;cannot connect\uFF0Ceven by Web browser\uFF0Cplease check that  there is some  problems\u3002&#x27;</span>,
 <span class="hljs-string">&#x27;I can access https://raw.githubusercontent.com/huggingface/datasets/1.7.0/datasets/glue/glue.py without problem...&#x27;</span>]`}}),_t=new A({props:{code:`comments_df = df.explode("comments", ignore_index=True)
comments_df.head(4)`,highlighted:`comments_df = df.explode(<span class="hljs-string">&quot;comments&quot;</span>, ignore_index=<span class="hljs-literal">True</span>)
comments_df.head(<span class="hljs-number">4</span>)`}}),gt=new A({props:{code:`from datasets import Dataset

comments_dataset = Dataset.from_pandas(comments_df)
comments_dataset`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset

comments_dataset = Dataset.from_pandas(comments_df)
comments_dataset`}}),vt=new A({props:{code:`Dataset({
    features: ['html_url', 'title', 'comments', 'body'],
    num_rows: 2842
})`,highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;html_url&#x27;</span>, <span class="hljs-string">&#x27;title&#x27;</span>, <span class="hljs-string">&#x27;comments&#x27;</span>, <span class="hljs-string">&#x27;body&#x27;</span>],
    num_rows: <span class="hljs-number">2842</span>
})`}}),Ve=new ed({props:{$$slots:{default:[dd]},$$scope:{ctx:G}}}),bt=new A({props:{code:`comments_dataset = comments_dataset.map(
    lambda x: {"comment_length": len(x["comments"].split())}
)`,highlighted:`comments_dataset = comments_dataset.<span class="hljs-built_in">map</span>(
    <span class="hljs-keyword">lambda</span> x: {<span class="hljs-string">&quot;comment_length&quot;</span>: <span class="hljs-built_in">len</span>(x[<span class="hljs-string">&quot;comments&quot;</span>].split())}
)`}}),$t=new A({props:{code:`comments_dataset = comments_dataset.filter(lambda x: x["comment_length"] > 15)
comments_dataset`,highlighted:`comments_dataset = comments_dataset.<span class="hljs-built_in">filter</span>(<span class="hljs-keyword">lambda</span> x: x[<span class="hljs-string">&quot;comment_length&quot;</span>] &gt; <span class="hljs-number">15</span>)
comments_dataset`}}),qt=new A({props:{code:`Dataset({
    features: ['html_url', 'title', 'comments', 'body', 'comment_length'],
    num_rows: 2098
})`,highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;html_url&#x27;</span>, <span class="hljs-string">&#x27;title&#x27;</span>, <span class="hljs-string">&#x27;comments&#x27;</span>, <span class="hljs-string">&#x27;body&#x27;</span>, <span class="hljs-string">&#x27;comment_length&#x27;</span>],
    num_rows: <span class="hljs-number">2098</span>
})`}}),Et=new A({props:{code:`def concatenate_text(examples):
    return {
        "text": examples["title"]
        + " \\n "
        + examples["body"]
        + " \\n "
        + examples["comments"]
    }


comments_dataset = comments_dataset.map(concatenate_text)`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">concatenate_text</span>(<span class="hljs-params">examples</span>):
    <span class="hljs-keyword">return</span> {
        <span class="hljs-string">&quot;text&quot;</span>: examples[<span class="hljs-string">&quot;title&quot;</span>]
        + <span class="hljs-string">&quot; \\n &quot;</span>
        + examples[<span class="hljs-string">&quot;body&quot;</span>]
        + <span class="hljs-string">&quot; \\n &quot;</span>
        + examples[<span class="hljs-string">&quot;comments&quot;</span>]
    }


comments_dataset = comments_dataset.<span class="hljs-built_in">map</span>(concatenate_text)`}}),xt=new Fn({});const Ji=[md,pd],Nt=[];function Qi(e,i){return e[0]==="pt"?0:1}ve=Qi(G),be=Nt[ve]=Ji[ve](G),yt=new A({props:{code:`def cls_pooling(model_output):
    return model_output.last_hidden_state[:, 0]`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">cls_pooling</span>(<span class="hljs-params">model_output</span>):
    <span class="hljs-keyword">return</span> model_output.last_hidden_state[:, <span class="hljs-number">0</span>]`}});const Xi=[hd,fd],Pt=[];function Ki(e,i){return e[0]==="pt"?0:1}$e=Ki(G),qe=Pt[$e]=Xi[$e](G),jt=new Fn({}),Ct=new A({props:{code:'embeddings_dataset.add_faiss_index(column="embeddings")',highlighted:'embeddings_dataset.add_faiss_index(column=<span class="hljs-string">&quot;embeddings&quot;</span>)'}});const Zi=[gd,_d],Mt=[];function eu(e,i){return e[0]==="pt"?0:1}return Ee=eu(G),xe=Mt[Ee]=Zi[Ee](G),Tt=new A({props:{code:`scores, samples = embeddings_dataset.get_nearest_examples(
    "embeddings", question_embedding, k=5
)`,highlighted:`scores, samples = embeddings_dataset.get_nearest_examples(
    <span class="hljs-string">&quot;embeddings&quot;</span>, question_embedding, k=<span class="hljs-number">5</span>
)`}}),At=new A({props:{code:`import pandas as pd

samples_df = pd.DataFrame.from_dict(samples)
samples_df["scores"] = scores
samples_df.sort_values("scores", ascending=False, inplace=True)`,highlighted:`<span class="hljs-keyword">import</span> pandas <span class="hljs-keyword">as</span> pd

samples_df = pd.DataFrame.from_dict(samples)
samples_df[<span class="hljs-string">&quot;scores&quot;</span>] = scores
samples_df.sort_values(<span class="hljs-string">&quot;scores&quot;</span>, ascending=<span class="hljs-literal">False</span>, inplace=<span class="hljs-literal">True</span>)`}}),St=new A({props:{code:`for _, row in samples_df.iterrows():
    print(f"COMMENT: {row.comments}")
    print(f"SCORE: {row.scores}")
    print(f"TITLE: {row.title}")
    print(f"URL: {row.html_url}")
    print("=" * 50)
    print()`,highlighted:`<span class="hljs-keyword">for</span> _, row <span class="hljs-keyword">in</span> samples_df.iterrows():
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;COMMENT: <span class="hljs-subst">{row.comments}</span>&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;SCORE: <span class="hljs-subst">{row.scores}</span>&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;TITLE: <span class="hljs-subst">{row.title}</span>&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;URL: <span class="hljs-subst">{row.html_url}</span>&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;=&quot;</span> * <span class="hljs-number">50</span>)
    <span class="hljs-built_in">print</span>()`}}),Ot=new A({props:{code:`"""
COMMENT: Requiring online connection is a deal breaker in some cases unfortunately so it'd be great if offline mode is added similar to how \`transformers\` loads models offline fine.

@mandubian's second bullet point suggests that there's a workaround allowing you to use your offline (custom?) dataset with \`datasets\`. Could you please elaborate on how that should look like?
SCORE: 25.505046844482422
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: The local dataset builders (csv, text , json and pandas) are now part of the \`datasets\` package since #1726 :)
You can now use them offline
\\\`\\\`\\\`python
datasets = load_dataset("text", data_files=data_files)
\\\`\\\`\\\`

We'll do a new release soon
SCORE: 24.555509567260742
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: I opened a PR that allows to reload modules that have already been loaded once even if there's no internet.

Let me know if you know other ways that can make the offline mode experience better. I'd be happy to add them :)

I already note the "freeze" modules option, to prevent local modules updates. It would be a cool feature.

----------

> @mandubian's second bullet point suggests that there's a workaround allowing you to use your offline (custom?) dataset with \`datasets\`. Could you please elaborate on how that should look like?

Indeed \`load_dataset\` allows to load remote dataset script (squad, glue, etc.) but also you own local ones.
For example if you have a dataset script at \`./my_dataset/my_dataset.py\` then you can do
\\\`\\\`\\\`python
load_dataset("./my_dataset")
\\\`\\\`\\\`
and the dataset script will generate your dataset once and for all.

----------

About I'm looking into having \`csv\`, \`json\`, \`text\`, \`pandas\` dataset builders already included in the \`datasets\` package, so that they are available offline by default, as opposed to the other datasets that require the script to be downloaded.
cf #1724
SCORE: 24.14896583557129
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: > here is my way to load a dataset offline, but it **requires** an online machine
>
> 1. (online machine)
>
> \`\`\`
>
> import datasets
>
> data = datasets.load_dataset(...)
>
> data.save_to_disk(/YOUR/DATASET/DIR)
>
> \`\`\`
>
> 2. copy the dir from online to the offline machine
>
> 3. (offline machine)
>
> \`\`\`
>
> import datasets
>
> data = datasets.load_from_disk(/SAVED/DATA/DIR)
>
> \`\`\`
>
>
>
> HTH.


SCORE: 22.893993377685547
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: here is my way to load a dataset offline, but it **requires** an online machine
1. (online machine)
\\\`\\\`\\\`
import datasets
data = datasets.load_dataset(...)
data.save_to_disk(/YOUR/DATASET/DIR)
\\\`\\\`\\\`
2. copy the dir from online to the offline machine
3. (offline machine)
\\\`\\\`\\\`
import datasets
data = datasets.load_from_disk(/SAVED/DATA/DIR)
\\\`\\\`\\\`

HTH.
SCORE: 22.406635284423828
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================
"""`,highlighted:`<span class="hljs-string">&quot;&quot;&quot;
COMMENT: Requiring online connection is a deal breaker in some cases unfortunately so it&#x27;d be great if offline mode is added similar to how \`transformers\` loads models offline fine.

@mandubian&#x27;s second bullet point suggests that there&#x27;s a workaround allowing you to use your offline (custom?) dataset with \`datasets\`. Could you please elaborate on how that should look like?
SCORE: 25.505046844482422
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: The local dataset builders (csv, text , json and pandas) are now part of the \`datasets\` package since #1726 :)
You can now use them offline
\\\`\\\`\\\`python
datasets = load_dataset(&quot;text&quot;, data_files=data_files)
\\\`\\\`\\\`

We&#x27;ll do a new release soon
SCORE: 24.555509567260742
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: I opened a PR that allows to reload modules that have already been loaded once even if there&#x27;s no internet.

Let me know if you know other ways that can make the offline mode experience better. I&#x27;d be happy to add them :)

I already note the &quot;freeze&quot; modules option, to prevent local modules updates. It would be a cool feature.

----------

&gt; @mandubian&#x27;s second bullet point suggests that there&#x27;s a workaround allowing you to use your offline (custom?) dataset with \`datasets\`. Could you please elaborate on how that should look like?

Indeed \`load_dataset\` allows to load remote dataset script (squad, glue, etc.) but also you own local ones.
For example if you have a dataset script at \`./my_dataset/my_dataset.py\` then you can do
\\\`\\\`\\\`python
load_dataset(&quot;./my_dataset&quot;)
\\\`\\\`\\\`
and the dataset script will generate your dataset once and for all.

----------

About I&#x27;m looking into having \`csv\`, \`json\`, \`text\`, \`pandas\` dataset builders already included in the \`datasets\` package, so that they are available offline by default, as opposed to the other datasets that require the script to be downloaded.
cf #1724
SCORE: 24.14896583557129
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: &gt; here is my way to load a dataset offline, but it **requires** an online machine
&gt;
&gt; 1. (online machine)
&gt;
&gt; \`\`\`
&gt;
&gt; import datasets
&gt;
&gt; data = datasets.load_dataset(...)
&gt;
&gt; data.save_to_disk(/YOUR/DATASET/DIR)
&gt;
&gt; \`\`\`
&gt;
&gt; 2. copy the dir from online to the offline machine
&gt;
&gt; 3. (offline machine)
&gt;
&gt; \`\`\`
&gt;
&gt; import datasets
&gt;
&gt; data = datasets.load_from_disk(/SAVED/DATA/DIR)
&gt;
&gt; \`\`\`
&gt;
&gt;
&gt;
&gt; HTH.


SCORE: 22.893993377685547
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: here is my way to load a dataset offline, but it **requires** an online machine
1. (online machine)
\\\`\\\`\\\`
import datasets
data = datasets.load_dataset(...)
data.save_to_disk(/YOUR/DATASET/DIR)
\\\`\\\`\\\`
2. copy the dir from online to the offline machine
3. (offline machine)
\\\`\\\`\\\`
import datasets
data = datasets.load_from_disk(/SAVED/DATA/DIR)
\\\`\\\`\\\`

HTH.
SCORE: 22.406635284423828
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================
&quot;&quot;&quot;</span>`}}),Xe=new ed({props:{$$slots:{default:[vd]},$$scope:{ctx:G}}}),{c(){m=o("meta"),q=d(),E(f.$$.fragment),j=d(),_=o("h1"),$=o("a"),S=o("span"),E(g.$$.fragment),C=d(),D=o("span"),z=n("Recherche s\xE9mantique avec FAISS"),O=d(),I.c(),N=d(),R=o("p"),Y=n("Dans "),P=o("a"),Q=n("section 5"),B=n(", nous avons cr\xE9\xE9 un jeu de donn\xE9es de probl\xE8mes et de commentaires GitHub \xE0 partir du d\xE9p\xF4t \u{1F917} "),U=o("em"),L=n("Datasets"),V=n(". Dans cette section, nous utilisons ces informations pour cr\xE9er un moteur de recherche qui peut nous aider \xE0 trouver des r\xE9ponses \xE0 nos questions les plus urgentes sur la biblioth\xE8que !"),c=d(),E(h.$$.fragment),F=d(),W=o("h2"),J=o("a"),K=o("span"),E(re.$$.fragment),Pe=d(),He=o("span"),ao=n("Utilisation des ench\xE2ssements pour la recherche s\xE9mantique"),Ln=d(),te=o("p"),oo=n("Comme nous l\u2019avons vu dans le "),Ft=o("a"),ro=n("chapitre 1"),lo=n(", les mod\xE8les de langage bas\xE9s sur les "),os=o("em"),io=n("transformers"),uo=n(" repr\xE9sentent chaque "),rs=o("em"),co=n("token"),po=n(" dans une \xE9tendue de texte sous la forme d\u2019un "),ls=o("em"),mo=n("ench\xE2ssement"),fo=n(". Il s\u2019av\xE8re que l\u2019on peut regrouper les ench\xE2ssements individuels pour cr\xE9er une repr\xE9sentation vectorielle pour des phrases enti\xE8res, des paragraphes ou (dans certains cas) des documents. Ces ench\xE2ssements peuvent ensuite \xEAtre utilis\xE9s pour trouver des documents similaires dans le corpus en calculant la similarit\xE9 du produit scalaire (ou une autre m\xE9trique de similarit\xE9) entre chaque ench\xE2ssement et en renvoyant les documents avec le plus grand chevauchement."),Hn=d(),Lt=o("p"),ho=n("Dans cette section, nous utilisons les ench\xE2ssements pour d\xE9velopper un moteur de recherche s\xE9mantique. Ces moteurs de recherche offrent plusieurs avantages par rapport aux approches conventionnelles bas\xE9es sur la correspondance des mots-cl\xE9s dans une requ\xEAte avec les documents."),zn=d(),Me=o("div"),st=o("img"),_o=d(),nt=o("img"),Un=d(),Re=o("h2"),ze=o("a"),is=o("span"),E(at.$$.fragment),go=d(),us=o("span"),vo=n("Chargement et pr\xE9paration du jeu de donn\xE9es"),Gn=d(),ke=o("p"),bo=n("La premi\xE8re chose que nous devons faire est de t\xE9l\xE9charger notre jeu de donn\xE9es de probl\xE8mes GitHub. Utilisons la biblioth\xE8que \u{1F917} "),cs=o("em"),$o=n("Hub"),qo=n(" pour r\xE9soudre l\u2019URL o\xF9 notre fichier est stock\xE9 sur le "),ds=o("em"),Eo=n("Hub"),xo=n(" d\u2019Hugging Face :"),Vn=d(),E(ot.$$.fragment),Yn=d(),we=o("p"),ko=n("Avec l\u2019URL stock\xE9 dans "),ps=o("code"),wo=n("data_files"),yo=n(", nous pouvons ensuite charger le jeu de donn\xE9es distant en utilisant la m\xE9thode introduite dans "),Ht=o("a"),jo=n("section 2"),Do=n(" :"),Bn=d(),E(rt.$$.fragment),Wn=d(),E(lt.$$.fragment),Jn=d(),X=o("p"),Co=n("Ici, nous avons sp\xE9cifi\xE9 l\u2019\xE9chantillon "),ms=o("code"),To=n("train"),Ao=n(" par d\xE9faut dans "),fs=o("code"),So=n("load_dataset()"),Oo=n(", de sorte que cela renvoie un "),hs=o("code"),Io=n("Dataset"),No=n(" au lieu d\u2019un "),_s=o("code"),Po=n("DatasetDict"),Mo=n(". La premi\xE8re chose \xE0 faire est de filtrer les "),gs=o("em"),Ro=n("pull requests"),Fo=n(" car celles-ci ont tendance \xE0 \xEAtre rarement utilis\xE9es pour r\xE9pondre aux requ\xEAtes des utilisateurs et introduiront du bruit dans notre moteur de recherche. Comme cela devrait \xEAtre familier maintenant, nous pouvons utiliser la fonction "),vs=o("code"),Lo=n("Dataset.filter()"),Ho=n(" pour exclure ces lignes de notre jeu de donn\xE9es. Pendant que nous y sommes, filtrons \xE9galement les lignes sans commentaires, car celles-ci ne fournissent aucune r\xE9ponse aux requ\xEAtes des utilisateurs :"),Qn=d(),E(it.$$.fragment),Xn=d(),E(ut.$$.fragment),Kn=d(),Z=o("p"),zo=n("Nous pouvons voir qu\u2019il y a beaucoup de colonnes dans notre jeu de donn\xE9es, dont la plupart n\u2019ont pas besoin de construire notre moteur de recherche. Du point de vue de la recherche, les colonnes les plus informatives sont "),bs=o("code"),Uo=n("title"),Go=n(", "),$s=o("code"),Vo=n("body"),Yo=n(" et "),qs=o("code"),Bo=n("comments"),Wo=n(", tandis que "),Es=o("code"),Jo=n("html_url"),Qo=n(" nous fournit un lien vers le probl\xE8me source. Utilisons la fonction "),xs=o("code"),Xo=n("Dataset.remove_columns()"),Ko=n(" pour supprimer le reste :"),Zn=d(),E(ct.$$.fragment),ea=d(),E(dt.$$.fragment),ta=d(),ee=o("p"),Zo=n("Pour cr\xE9er nos ench\xE2ssements, nous ajoutons \xE0 chaque commentaire le titre et le corps du probl\xE8me, car ces champs contiennent des informations contextuelles utiles. \xC9tant donn\xE9 que notre colonne "),ks=o("code"),er=n("comments"),tr=n(" est actuellement une liste de commentaires pour chaque probl\xE8me, nous devons \xAB \xE9clater \xBB la colonne afin que chaque ligne se compose d\u2019un "),ws=o("em"),sr=n("tuple"),nr=d(),ys=o("code"),ar=n("(html_url, title, body, comment)"),or=n(". Dans Pandas, nous pouvons le faire avec la fonction "),pt=o("a"),js=o("code"),rr=n("DataFrame.explode()"),lr=n(", qui cr\xE9e une nouvelle ligne pour chaque \xE9l\xE9ment dans une colonne de type liste, tout en r\xE9pliquant toutes les autres valeurs de colonne. Pour voir cela en action, passons d\u2019abord au format "),Ds=o("code"),ir=n("DataFrame"),ur=n(" de Pandas :"),sa=d(),E(mt.$$.fragment),na=d(),Ue=o("p"),cr=n("Si nous inspectons la premi\xE8re ligne de ce "),Cs=o("code"),dr=n("DataFrame"),pr=n(", nous pouvons voir qu\u2019il y a quatre commentaires associ\xE9s \xE0 ce probl\xE8me :"),aa=d(),E(ft.$$.fragment),oa=d(),E(ht.$$.fragment),ra=d(),Ge=o("p"),mr=n("Lorsque nous d\xE9composons "),Ts=o("code"),fr=n("df"),hr=n(", nous nous attendons \xE0 obtenir une ligne pour chacun de ces commentaires. V\xE9rifions si c\u2019est le cas :"),la=d(),E(_t.$$.fragment),ia=d(),le=o("table"),As=o("thead"),se=o("tr"),ua=o("th"),_r=d(),Ss=o("th"),gr=n("html_url"),vr=d(),Os=o("th"),br=n("title"),$r=d(),Is=o("th"),qr=n("comments"),Er=d(),Ns=o("th"),xr=n("body"),kr=d(),ge=o("tbody"),ie=o("tr"),Ps=o("th"),wr=n("0"),yr=d(),Ms=o("td"),jr=n("https://github.com/huggingface/datasets/issues/2787"),Dr=d(),Rs=o("td"),Cr=n("ConnectionError: Couldn't reach https://raw.githubusercontent.com"),Tr=d(),Fs=o("td"),Ar=n("the bug code locate in \uFF1A\\r\\n    if data_args.task_name is not None..."),Sr=d(),Ls=o("td"),Or=n("Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),Ir=d(),ue=o("tr"),Hs=o("th"),Nr=n("1"),Pr=d(),zs=o("td"),Mr=n("https://github.com/huggingface/datasets/issues/2787"),Rr=d(),Us=o("td"),Fr=n("ConnectionError: Couldn't reach https://raw.githubusercontent.com"),Lr=d(),Gs=o("td"),Hr=n("Hi @jinec,\\r\\n\\r\\nFrom time to time we get this kind of `ConnectionError` coming from the github.com website: https://raw.githubusercontent.com..."),zr=d(),Vs=o("td"),Ur=n("Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),Gr=d(),ce=o("tr"),Ys=o("th"),Vr=n("2"),Yr=d(),Bs=o("td"),Br=n("https://github.com/huggingface/datasets/issues/2787"),Wr=d(),Ws=o("td"),Jr=n("ConnectionError: Couldn't reach https://raw.githubusercontent.com"),Qr=d(),Js=o("td"),Xr=n("cannot connect\uFF0Ceven by Web browser\uFF0Cplease check that  there is some  problems\u3002"),Kr=d(),Qs=o("td"),Zr=n("Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),el=d(),de=o("tr"),Xs=o("th"),tl=n("3"),sl=d(),Ks=o("td"),nl=n("https://github.com/huggingface/datasets/issues/2787"),al=d(),Zs=o("td"),ol=n("ConnectionError: Couldn't reach https://raw.githubusercontent.com"),rl=d(),en=o("td"),ll=n("I can access https://raw.githubusercontent.com/huggingface/datasets/1.7.0/datasets/glue/glue.py without problem..."),il=d(),tn=o("td"),ul=n("Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),ca=d(),pe=o("p"),cl=n("G\xE9nial, nous pouvons voir que les lignes ont \xE9t\xE9 r\xE9pliqu\xE9es, avec la colonne "),sn=o("code"),dl=n("comments"),pl=n(" contenant les commentaires individuels ! Maintenant que nous en avons fini avec Pandas, nous pouvons rapidement revenir \xE0 un "),nn=o("code"),ml=n("Dataset"),fl=n(" en chargeant le "),an=o("code"),hl=n("DataFrame"),_l=n(" en m\xE9moire :"),da=d(),E(gt.$$.fragment),pa=d(),E(vt.$$.fragment),ma=d(),zt=o("p"),gl=n("D\u2019accord, cela nous a donn\xE9 quelques milliers de commentaires avec lesquels travailler !"),fa=d(),E(Ve.$$.fragment),ha=d(),Ye=o("p"),vl=n("Maintenant que nous avons un commentaire par ligne, cr\xE9ons une nouvelle colonne "),on=o("code"),bl=n("comments_length"),$l=n(" contenant le nombre de mots par commentaire :"),_a=d(),E(bt.$$.fragment),ga=d(),Ut=o("p"),ql=n("Nous pouvons utiliser cette nouvelle colonne pour filtrer les commentaires courts incluant g\xE9n\xE9ralement des \xE9l\xE9ments tels que \xAB cc @lewtun \xBB ou \xAB Merci ! \xBB qui ne sont pas pertinents pour notre moteur de recherche. Il n\u2019y a pas de nombre pr\xE9cis \xE0 s\xE9lectionner pour le filtre mais 15 mots semblent \xEAtre un bon d\xE9but :"),va=d(),E($t.$$.fragment),ba=d(),E(qt.$$.fragment),$a=d(),ye=o("p"),El=n("Apr\xE8s avoir un peu nettoy\xE9 notre jeu de donn\xE9es, concat\xE9nons le titre, la description et les commentaires du probl\xE8me dans une nouvelle colonne "),rn=o("code"),xl=n("text"),kl=n(". Comme d\u2019habitude, nous allons \xE9crire une fonction simple que nous pouvons passer \xE0 "),ln=o("code"),wl=n("Dataset.map()"),yl=n(" :"),qa=d(),E(Et.$$.fragment),Ea=d(),Gt=o("p"),jl=n("Nous sommes enfin pr\xEAts \xE0 cr\xE9er des ench\xE2ssements ! Jetons un coup d\u2019\u0153il."),xa=d(),Fe=o("h2"),Be=o("a"),un=o("span"),E(xt.$$.fragment),Dl=d(),cn=o("span"),Cl=n("Cr\xE9ation d\u2019ench\xE2ssements pour les textes"),ka=d(),M=o("p"),Tl=n("Nous avons vu dans "),Vt=o("a"),Al=n("chapitre 2"),Sl=n(" que nous pouvons obtenir des ench\xE2ssements de "),dn=o("em"),Ol=n("tokens"),Il=n(" en utilisant la classe "),pn=o("code"),Nl=n("AutoModel"),Pl=n(". Tout ce que nous avons \xE0 faire est de choisir un "),mn=o("em"),Ml=n("checkpoint"),Rl=n(" appropri\xE9 \xE0 partir duquel charger le mod\xE8le. Heureusement, il existe une biblioth\xE8que appel\xE9e "),fn=o("code"),Fl=n("sentence-transformers"),Ll=n(" d\xE9di\xE9e \xE0 la cr\xE9ation d\u2019ench\xE2ssements. Comme d\xE9crit dans la "),kt=o("a"),Hl=n("documentation de la biblioth\xE8que"),zl=n(", notre cas d\u2019utilisation est un exemple de "),hn=o("em"),Ul=n("recherche s\xE9mantique asym\xE9trique"),Gl=n(". En effet, nous avons une requ\xEAte courte dont nous aimerions trouver la r\xE9ponse dans un document plus long, par exemple un commentaire \xE0 un probl\xE8me. Le "),wt=o("a"),Vl=n("tableau de pr\xE9sentation des mod\xE8les"),Yl=n(" de la documentation indique que le "),_n=o("em"),Bl=n("checkpoint"),Wl=d(),gn=o("code"),Jl=n("multi-qa-mpnet-base-dot-v1"),Ql=n(" a les meilleures performances pour la recherche s\xE9mantique. Utilisons donc le pour notre application. Nous allons \xE9galement charger le "),vn=o("em"),Xl=n("tokenizer"),Kl=n(" en utilisant le m\xEAme "),bn=o("em"),Zl=n("checkpoint"),ei=n(" :"),wa=d(),be.c(),Yt=d(),ne=o("p"),ti=n("Comme nous l\u2019avons mentionn\xE9 pr\xE9c\xE9demment, nous aimerions repr\xE9senter chaque entr\xE9e dans notre corpus de probl\xE8mes GitHub comme un vecteur unique. Nous devons donc regrouper ou faire la moyenne de nos ench\xE2ssements de "),$n=o("em"),si=n("tokens"),ni=n(" d\u2019une mani\xE8re ou d\u2019une autre. Une approche populaire consiste \xE0 effectuer un "),qn=o("em"),ai=n("regroupement CLS"),oi=n(" sur les sorties de notre mod\xE8le, o\xF9 nous collectons simplement le dernier \xE9tat cach\xE9 pour le "),En=o("em"),ri=n("token"),li=n(" sp\xE9cial "),xn=o("code"),ii=n("[CLS]"),ui=n(". La fonction suivante fait \xE7a pour nous :"),ya=d(),E(yt.$$.fragment),ja=d(),Bt=o("p"),ci=n("Ensuite, nous allons cr\xE9er une fonction utile qui va tokeniser une liste de documents, placer les tenseurs dans le GPU, les donner au mod\xE8le et enfin appliquer le regroupement CLS aux sorties :"),Da=d(),qe.c(),Wt=d(),We=o("p"),di=n("Notez que nous avons converti les ench\xE2ssements en tableaux NumPy. C\u2019est parce que \u{1F917} "),kn=o("em"),pi=n("Datasets"),mi=n(" n\xE9cessite ce format lorsque nous essayons de les indexer avec FAISS, ce que nous ferons ensuite."),Ca=d(),Le=o("h2"),Je=o("a"),wn=o("span"),E(jt.$$.fragment),fi=d(),yn=o("span"),hi=n("Utilisation de FAISS pour une recherche de similarit\xE9 efficace"),Ta=d(),ae=o("p"),_i=n("Maintenant que nous avons un jeu de donn\xE9es d\u2019incorporations, nous avons besoin d\u2019un moyen de les rechercher. Pour ce faire, nous utiliserons une structure de donn\xE9es sp\xE9ciale dans \u{1F917} "),jn=o("em"),gi=n("Datasets"),vi=n(" appel\xE9e "),Dn=o("em"),bi=n("FAISS index"),$i=n(". "),Dt=o("a"),qi=n("FAISS"),Ei=n(" (abr\xE9viation de "),Cn=o("em"),xi=n("Facebook AI Similarity Search"),ki=n(") est une biblioth\xE8que qui fournit des algorithmes efficaces pour rechercher et regrouper rapidement des vecteurs d\u2019int\xE9gration."),Aa=d(),me=o("p"),wi=n("L\u2019id\xE9e de base derri\xE8re FAISS est de cr\xE9er une structure de donn\xE9es sp\xE9ciale appel\xE9e un "),Tn=o("em"),yi=n("index"),ji=n(" qui permet de trouver quels plongements sont similaires \xE0 un plongement d\u2019entr\xE9e. Cr\xE9er un index FAISS dans \u{1F917} "),An=o("em"),Di=n("Datasets"),Ci=n(" est simple \u2014 nous utilisons la fonction "),Sn=o("code"),Ti=n("Dataset.add_faiss_index()"),Ai=n(" et sp\xE9cifions quelle colonne de notre jeu de donn\xE9es nous aimerions indexer :"),Sa=d(),E(Ct.$$.fragment),Oa=d(),Qe=o("p"),Si=n("Nous pouvons maintenant effectuer des requ\xEAtes sur cet index en effectuant une recherche des voisins les plus proches avec la fonction "),On=o("code"),Oi=n("Dataset.get_nearest_examples()"),Ii=n(". Testons cela en ench\xE2ssant d\u2019abord une question comme suit :"),Ia=d(),xe.c(),Jt=d(),Qt=o("p"),Ni=n("Tout comme avec les documents, nous avons maintenant un vecteur de 768 dimensions repr\xE9sentant la requ\xEAte. Nous pouvons le comparer \xE0 l\u2019ensemble du corpus pour trouver les ench\xE2ssements les plus similaires :"),Na=d(),E(Tt.$$.fragment),Pa=d(),fe=o("p"),Pi=n("La fonction "),In=o("code"),Mi=n("Dataset.get_nearest_examples()"),Ri=n(" renvoie un "),Nn=o("em"),Fi=n("tuple"),Li=n(" de scores qui classent le chevauchement entre la requ\xEAte et le document, et un jeu correspondant d\u2019\xE9chantillons (ici, les 5 meilleures correspondances). Collectons-les dans un "),Pn=o("code"),Hi=n("pandas.DataFrame"),zi=n(" afin de pouvoir les trier facilement :"),Ma=d(),E(At.$$.fragment),Ra=d(),Xt=o("p"),Ui=n("Nous pouvons maintenant parcourir les premi\xE8res lignes pour voir dans quelle mesure notre requ\xEAte correspond aux commentaires disponibles :"),Fa=d(),E(St.$$.fragment),La=d(),E(Ot.$$.fragment),Ha=d(),Kt=o("p"),Gi=n("Pas mal ! Notre deuxi\xE8me r\xE9sultat semble correspondre \xE0 la requ\xEAte."),za=d(),E(Xe.$$.fragment),this.h()},l(e){const i=od('[data-svelte="svelte-1phssyn"]',document.head);m=r(i,"META",{name:!0,content:!0}),i.forEach(s),q=p(e),x(f.$$.fragment,e),j=p(e),_=r(e,"H1",{class:!0});var Rt=l(_);$=r(Rt,"A",{id:!0,class:!0,href:!0});var Zt=l($);S=r(Zt,"SPAN",{});var Mn=l(S);x(g.$$.fragment,Mn),Mn.forEach(s),Zt.forEach(s),C=p(Rt),D=r(Rt,"SPAN",{});var es=l(D);z=a(es,"Recherche s\xE9mantique avec FAISS"),es.forEach(s),Rt.forEach(s),O=p(e),I.l(e),N=p(e),R=r(e,"P",{});var je=l(R);Y=a(je,"Dans "),P=r(je,"A",{href:!0});var ts=l(P);Q=a(ts,"section 5"),ts.forEach(s),B=a(je,", nous avons cr\xE9\xE9 un jeu de donn\xE9es de probl\xE8mes et de commentaires GitHub \xE0 partir du d\xE9p\xF4t \u{1F917} "),U=r(je,"EM",{});var Rn=l(U);L=a(Rn,"Datasets"),Rn.forEach(s),V=a(je,". Dans cette section, nous utilisons ces informations pour cr\xE9er un moteur de recherche qui peut nous aider \xE0 trouver des r\xE9ponses \xE0 nos questions les plus urgentes sur la biblioth\xE8que !"),je.forEach(s),c=p(e),x(h.$$.fragment,e),F=p(e),W=r(e,"H2",{class:!0});var Ga=l(W);J=r(Ga,"A",{id:!0,class:!0,href:!0});var tu=l(J);K=r(tu,"SPAN",{});var su=l(K);x(re.$$.fragment,su),su.forEach(s),tu.forEach(s),Pe=p(Ga),He=r(Ga,"SPAN",{});var nu=l(He);ao=a(nu,"Utilisation des ench\xE2ssements pour la recherche s\xE9mantique"),nu.forEach(s),Ga.forEach(s),Ln=p(e),te=r(e,"P",{});var De=l(te);oo=a(De,"Comme nous l\u2019avons vu dans le "),Ft=r(De,"A",{href:!0});var au=l(Ft);ro=a(au,"chapitre 1"),au.forEach(s),lo=a(De,", les mod\xE8les de langage bas\xE9s sur les "),os=r(De,"EM",{});var ou=l(os);io=a(ou,"transformers"),ou.forEach(s),uo=a(De," repr\xE9sentent chaque "),rs=r(De,"EM",{});var ru=l(rs);co=a(ru,"token"),ru.forEach(s),po=a(De," dans une \xE9tendue de texte sous la forme d\u2019un "),ls=r(De,"EM",{});var lu=l(ls);mo=a(lu,"ench\xE2ssement"),lu.forEach(s),fo=a(De,". Il s\u2019av\xE8re que l\u2019on peut regrouper les ench\xE2ssements individuels pour cr\xE9er une repr\xE9sentation vectorielle pour des phrases enti\xE8res, des paragraphes ou (dans certains cas) des documents. Ces ench\xE2ssements peuvent ensuite \xEAtre utilis\xE9s pour trouver des documents similaires dans le corpus en calculant la similarit\xE9 du produit scalaire (ou une autre m\xE9trique de similarit\xE9) entre chaque ench\xE2ssement et en renvoyant les documents avec le plus grand chevauchement."),De.forEach(s),Hn=p(e),Lt=r(e,"P",{});var iu=l(Lt);ho=a(iu,"Dans cette section, nous utilisons les ench\xE2ssements pour d\xE9velopper un moteur de recherche s\xE9mantique. Ces moteurs de recherche offrent plusieurs avantages par rapport aux approches conventionnelles bas\xE9es sur la correspondance des mots-cl\xE9s dans une requ\xEAte avec les documents."),iu.forEach(s),zn=p(e),Me=r(e,"DIV",{class:!0});var Va=l(Me);st=r(Va,"IMG",{class:!0,src:!0,alt:!0}),_o=p(Va),nt=r(Va,"IMG",{class:!0,src:!0,alt:!0}),Va.forEach(s),Un=p(e),Re=r(e,"H2",{class:!0});var Ya=l(Re);ze=r(Ya,"A",{id:!0,class:!0,href:!0});var uu=l(ze);is=r(uu,"SPAN",{});var cu=l(is);x(at.$$.fragment,cu),cu.forEach(s),uu.forEach(s),go=p(Ya),us=r(Ya,"SPAN",{});var du=l(us);vo=a(du,"Chargement et pr\xE9paration du jeu de donn\xE9es"),du.forEach(s),Ya.forEach(s),Gn=p(e),ke=r(e,"P",{});var ss=l(ke);bo=a(ss,"La premi\xE8re chose que nous devons faire est de t\xE9l\xE9charger notre jeu de donn\xE9es de probl\xE8mes GitHub. Utilisons la biblioth\xE8que \u{1F917} "),cs=r(ss,"EM",{});var pu=l(cs);$o=a(pu,"Hub"),pu.forEach(s),qo=a(ss," pour r\xE9soudre l\u2019URL o\xF9 notre fichier est stock\xE9 sur le "),ds=r(ss,"EM",{});var mu=l(ds);Eo=a(mu,"Hub"),mu.forEach(s),xo=a(ss," d\u2019Hugging Face :"),ss.forEach(s),Vn=p(e),x(ot.$$.fragment,e),Yn=p(e),we=r(e,"P",{});var ns=l(we);ko=a(ns,"Avec l\u2019URL stock\xE9 dans "),ps=r(ns,"CODE",{});var fu=l(ps);wo=a(fu,"data_files"),fu.forEach(s),yo=a(ns,", nous pouvons ensuite charger le jeu de donn\xE9es distant en utilisant la m\xE9thode introduite dans "),Ht=r(ns,"A",{href:!0});var hu=l(Ht);jo=a(hu,"section 2"),hu.forEach(s),Do=a(ns," :"),ns.forEach(s),Bn=p(e),x(rt.$$.fragment,e),Wn=p(e),x(lt.$$.fragment,e),Jn=p(e),X=r(e,"P",{});var oe=l(X);Co=a(oe,"Ici, nous avons sp\xE9cifi\xE9 l\u2019\xE9chantillon "),ms=r(oe,"CODE",{});var _u=l(ms);To=a(_u,"train"),_u.forEach(s),Ao=a(oe," par d\xE9faut dans "),fs=r(oe,"CODE",{});var gu=l(fs);So=a(gu,"load_dataset()"),gu.forEach(s),Oo=a(oe,", de sorte que cela renvoie un "),hs=r(oe,"CODE",{});var vu=l(hs);Io=a(vu,"Dataset"),vu.forEach(s),No=a(oe," au lieu d\u2019un "),_s=r(oe,"CODE",{});var bu=l(_s);Po=a(bu,"DatasetDict"),bu.forEach(s),Mo=a(oe,". La premi\xE8re chose \xE0 faire est de filtrer les "),gs=r(oe,"EM",{});var $u=l(gs);Ro=a($u,"pull requests"),$u.forEach(s),Fo=a(oe," car celles-ci ont tendance \xE0 \xEAtre rarement utilis\xE9es pour r\xE9pondre aux requ\xEAtes des utilisateurs et introduiront du bruit dans notre moteur de recherche. Comme cela devrait \xEAtre familier maintenant, nous pouvons utiliser la fonction "),vs=r(oe,"CODE",{});var qu=l(vs);Lo=a(qu,"Dataset.filter()"),qu.forEach(s),Ho=a(oe," pour exclure ces lignes de notre jeu de donn\xE9es. Pendant que nous y sommes, filtrons \xE9galement les lignes sans commentaires, car celles-ci ne fournissent aucune r\xE9ponse aux requ\xEAtes des utilisateurs :"),oe.forEach(s),Qn=p(e),x(it.$$.fragment,e),Xn=p(e),x(ut.$$.fragment,e),Kn=p(e),Z=r(e,"P",{});var he=l(Z);zo=a(he,"Nous pouvons voir qu\u2019il y a beaucoup de colonnes dans notre jeu de donn\xE9es, dont la plupart n\u2019ont pas besoin de construire notre moteur de recherche. Du point de vue de la recherche, les colonnes les plus informatives sont "),bs=r(he,"CODE",{});var Eu=l(bs);Uo=a(Eu,"title"),Eu.forEach(s),Go=a(he,", "),$s=r(he,"CODE",{});var xu=l($s);Vo=a(xu,"body"),xu.forEach(s),Yo=a(he," et "),qs=r(he,"CODE",{});var ku=l(qs);Bo=a(ku,"comments"),ku.forEach(s),Wo=a(he,", tandis que "),Es=r(he,"CODE",{});var wu=l(Es);Jo=a(wu,"html_url"),wu.forEach(s),Qo=a(he," nous fournit un lien vers le probl\xE8me source. Utilisons la fonction "),xs=r(he,"CODE",{});var yu=l(xs);Xo=a(yu,"Dataset.remove_columns()"),yu.forEach(s),Ko=a(he," pour supprimer le reste :"),he.forEach(s),Zn=p(e),x(ct.$$.fragment,e),ea=p(e),x(dt.$$.fragment,e),ta=p(e),ee=r(e,"P",{});var _e=l(ee);Zo=a(_e,"Pour cr\xE9er nos ench\xE2ssements, nous ajoutons \xE0 chaque commentaire le titre et le corps du probl\xE8me, car ces champs contiennent des informations contextuelles utiles. \xC9tant donn\xE9 que notre colonne "),ks=r(_e,"CODE",{});var ju=l(ks);er=a(ju,"comments"),ju.forEach(s),tr=a(_e," est actuellement une liste de commentaires pour chaque probl\xE8me, nous devons \xAB \xE9clater \xBB la colonne afin que chaque ligne se compose d\u2019un "),ws=r(_e,"EM",{});var Du=l(ws);sr=a(Du,"tuple"),Du.forEach(s),nr=p(_e),ys=r(_e,"CODE",{});var Cu=l(ys);ar=a(Cu,"(html_url, title, body, comment)"),Cu.forEach(s),or=a(_e,". Dans Pandas, nous pouvons le faire avec la fonction "),pt=r(_e,"A",{href:!0,rel:!0});var Tu=l(pt);js=r(Tu,"CODE",{});var Au=l(js);rr=a(Au,"DataFrame.explode()"),Au.forEach(s),Tu.forEach(s),lr=a(_e,", qui cr\xE9e une nouvelle ligne pour chaque \xE9l\xE9ment dans une colonne de type liste, tout en r\xE9pliquant toutes les autres valeurs de colonne. Pour voir cela en action, passons d\u2019abord au format "),Ds=r(_e,"CODE",{});var Su=l(Ds);ir=a(Su,"DataFrame"),Su.forEach(s),ur=a(_e," de Pandas :"),_e.forEach(s),sa=p(e),x(mt.$$.fragment,e),na=p(e),Ue=r(e,"P",{});var Ba=l(Ue);cr=a(Ba,"Si nous inspectons la premi\xE8re ligne de ce "),Cs=r(Ba,"CODE",{});var Ou=l(Cs);dr=a(Ou,"DataFrame"),Ou.forEach(s),pr=a(Ba,", nous pouvons voir qu\u2019il y a quatre commentaires associ\xE9s \xE0 ce probl\xE8me :"),Ba.forEach(s),aa=p(e),x(ft.$$.fragment,e),oa=p(e),x(ht.$$.fragment,e),ra=p(e),Ge=r(e,"P",{});var Wa=l(Ge);mr=a(Wa,"Lorsque nous d\xE9composons "),Ts=r(Wa,"CODE",{});var Iu=l(Ts);fr=a(Iu,"df"),Iu.forEach(s),hr=a(Wa,", nous nous attendons \xE0 obtenir une ligne pour chacun de ces commentaires. V\xE9rifions si c\u2019est le cas :"),Wa.forEach(s),la=p(e),x(_t.$$.fragment,e),ia=p(e),le=r(e,"TABLE",{border:!0,class:!0,style:!0});var Ja=l(le);As=r(Ja,"THEAD",{});var Nu=l(As);se=r(Nu,"TR",{style:!0});var Ce=l(se);ua=r(Ce,"TH",{}),l(ua).forEach(s),_r=p(Ce),Ss=r(Ce,"TH",{});var Pu=l(Ss);gr=a(Pu,"html_url"),Pu.forEach(s),vr=p(Ce),Os=r(Ce,"TH",{});var Mu=l(Os);br=a(Mu,"title"),Mu.forEach(s),$r=p(Ce),Is=r(Ce,"TH",{});var Ru=l(Is);qr=a(Ru,"comments"),Ru.forEach(s),Er=p(Ce),Ns=r(Ce,"TH",{});var Fu=l(Ns);xr=a(Fu,"body"),Fu.forEach(s),Ce.forEach(s),Nu.forEach(s),kr=p(Ja),ge=r(Ja,"TBODY",{});var Ke=l(ge);ie=r(Ke,"TR",{});var Te=l(ie);Ps=r(Te,"TH",{});var Lu=l(Ps);wr=a(Lu,"0"),Lu.forEach(s),yr=p(Te),Ms=r(Te,"TD",{});var Hu=l(Ms);jr=a(Hu,"https://github.com/huggingface/datasets/issues/2787"),Hu.forEach(s),Dr=p(Te),Rs=r(Te,"TD",{});var zu=l(Rs);Cr=a(zu,"ConnectionError: Couldn't reach https://raw.githubusercontent.com"),zu.forEach(s),Tr=p(Te),Fs=r(Te,"TD",{});var Uu=l(Fs);Ar=a(Uu,"the bug code locate in \uFF1A\\r\\n    if data_args.task_name is not None..."),Uu.forEach(s),Sr=p(Te),Ls=r(Te,"TD",{});var Gu=l(Ls);Or=a(Gu,"Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),Gu.forEach(s),Te.forEach(s),Ir=p(Ke),ue=r(Ke,"TR",{});var Ae=l(ue);Hs=r(Ae,"TH",{});var Vu=l(Hs);Nr=a(Vu,"1"),Vu.forEach(s),Pr=p(Ae),zs=r(Ae,"TD",{});var Yu=l(zs);Mr=a(Yu,"https://github.com/huggingface/datasets/issues/2787"),Yu.forEach(s),Rr=p(Ae),Us=r(Ae,"TD",{});var Bu=l(Us);Fr=a(Bu,"ConnectionError: Couldn't reach https://raw.githubusercontent.com"),Bu.forEach(s),Lr=p(Ae),Gs=r(Ae,"TD",{});var Wu=l(Gs);Hr=a(Wu,"Hi @jinec,\\r\\n\\r\\nFrom time to time we get this kind of `ConnectionError` coming from the github.com website: https://raw.githubusercontent.com..."),Wu.forEach(s),zr=p(Ae),Vs=r(Ae,"TD",{});var Ju=l(Vs);Ur=a(Ju,"Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),Ju.forEach(s),Ae.forEach(s),Gr=p(Ke),ce=r(Ke,"TR",{});var Se=l(ce);Ys=r(Se,"TH",{});var Qu=l(Ys);Vr=a(Qu,"2"),Qu.forEach(s),Yr=p(Se),Bs=r(Se,"TD",{});var Xu=l(Bs);Br=a(Xu,"https://github.com/huggingface/datasets/issues/2787"),Xu.forEach(s),Wr=p(Se),Ws=r(Se,"TD",{});var Ku=l(Ws);Jr=a(Ku,"ConnectionError: Couldn't reach https://raw.githubusercontent.com"),Ku.forEach(s),Qr=p(Se),Js=r(Se,"TD",{});var Zu=l(Js);Xr=a(Zu,"cannot connect\uFF0Ceven by Web browser\uFF0Cplease check that  there is some  problems\u3002"),Zu.forEach(s),Kr=p(Se),Qs=r(Se,"TD",{});var ec=l(Qs);Zr=a(ec,"Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),ec.forEach(s),Se.forEach(s),el=p(Ke),de=r(Ke,"TR",{});var Oe=l(de);Xs=r(Oe,"TH",{});var tc=l(Xs);tl=a(tc,"3"),tc.forEach(s),sl=p(Oe),Ks=r(Oe,"TD",{});var sc=l(Ks);nl=a(sc,"https://github.com/huggingface/datasets/issues/2787"),sc.forEach(s),al=p(Oe),Zs=r(Oe,"TD",{});var nc=l(Zs);ol=a(nc,"ConnectionError: Couldn't reach https://raw.githubusercontent.com"),nc.forEach(s),rl=p(Oe),en=r(Oe,"TD",{});var ac=l(en);ll=a(ac,"I can access https://raw.githubusercontent.com/huggingface/datasets/1.7.0/datasets/glue/glue.py without problem..."),ac.forEach(s),il=p(Oe),tn=r(Oe,"TD",{});var oc=l(tn);ul=a(oc,"Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),oc.forEach(s),Oe.forEach(s),Ke.forEach(s),Ja.forEach(s),ca=p(e),pe=r(e,"P",{});var Ze=l(pe);cl=a(Ze,"G\xE9nial, nous pouvons voir que les lignes ont \xE9t\xE9 r\xE9pliqu\xE9es, avec la colonne "),sn=r(Ze,"CODE",{});var rc=l(sn);dl=a(rc,"comments"),rc.forEach(s),pl=a(Ze," contenant les commentaires individuels ! Maintenant que nous en avons fini avec Pandas, nous pouvons rapidement revenir \xE0 un "),nn=r(Ze,"CODE",{});var lc=l(nn);ml=a(lc,"Dataset"),lc.forEach(s),fl=a(Ze," en chargeant le "),an=r(Ze,"CODE",{});var ic=l(an);hl=a(ic,"DataFrame"),ic.forEach(s),_l=a(Ze," en m\xE9moire :"),Ze.forEach(s),da=p(e),x(gt.$$.fragment,e),pa=p(e),x(vt.$$.fragment,e),ma=p(e),zt=r(e,"P",{});var uc=l(zt);gl=a(uc,"D\u2019accord, cela nous a donn\xE9 quelques milliers de commentaires avec lesquels travailler !"),uc.forEach(s),fa=p(e),x(Ve.$$.fragment,e),ha=p(e),Ye=r(e,"P",{});var Qa=l(Ye);vl=a(Qa,"Maintenant que nous avons un commentaire par ligne, cr\xE9ons une nouvelle colonne "),on=r(Qa,"CODE",{});var cc=l(on);bl=a(cc,"comments_length"),cc.forEach(s),$l=a(Qa," contenant le nombre de mots par commentaire :"),Qa.forEach(s),_a=p(e),x(bt.$$.fragment,e),ga=p(e),Ut=r(e,"P",{});var dc=l(Ut);ql=a(dc,"Nous pouvons utiliser cette nouvelle colonne pour filtrer les commentaires courts incluant g\xE9n\xE9ralement des \xE9l\xE9ments tels que \xAB cc @lewtun \xBB ou \xAB Merci ! \xBB qui ne sont pas pertinents pour notre moteur de recherche. Il n\u2019y a pas de nombre pr\xE9cis \xE0 s\xE9lectionner pour le filtre mais 15 mots semblent \xEAtre un bon d\xE9but :"),dc.forEach(s),va=p(e),x($t.$$.fragment,e),ba=p(e),x(qt.$$.fragment,e),$a=p(e),ye=r(e,"P",{});var as=l(ye);El=a(as,"Apr\xE8s avoir un peu nettoy\xE9 notre jeu de donn\xE9es, concat\xE9nons le titre, la description et les commentaires du probl\xE8me dans une nouvelle colonne "),rn=r(as,"CODE",{});var pc=l(rn);xl=a(pc,"text"),pc.forEach(s),kl=a(as,". Comme d\u2019habitude, nous allons \xE9crire une fonction simple que nous pouvons passer \xE0 "),ln=r(as,"CODE",{});var mc=l(ln);wl=a(mc,"Dataset.map()"),mc.forEach(s),yl=a(as," :"),as.forEach(s),qa=p(e),x(Et.$$.fragment,e),Ea=p(e),Gt=r(e,"P",{});var fc=l(Gt);jl=a(fc,"Nous sommes enfin pr\xEAts \xE0 cr\xE9er des ench\xE2ssements ! Jetons un coup d\u2019\u0153il."),fc.forEach(s),xa=p(e),Fe=r(e,"H2",{class:!0});var Xa=l(Fe);Be=r(Xa,"A",{id:!0,class:!0,href:!0});var hc=l(Be);un=r(hc,"SPAN",{});var _c=l(un);x(xt.$$.fragment,_c),_c.forEach(s),hc.forEach(s),Dl=p(Xa),cn=r(Xa,"SPAN",{});var gc=l(cn);Cl=a(gc,"Cr\xE9ation d\u2019ench\xE2ssements pour les textes"),gc.forEach(s),Xa.forEach(s),ka=p(e),M=r(e,"P",{});var H=l(M);Tl=a(H,"Nous avons vu dans "),Vt=r(H,"A",{href:!0});var vc=l(Vt);Al=a(vc,"chapitre 2"),vc.forEach(s),Sl=a(H," que nous pouvons obtenir des ench\xE2ssements de "),dn=r(H,"EM",{});var bc=l(dn);Ol=a(bc,"tokens"),bc.forEach(s),Il=a(H," en utilisant la classe "),pn=r(H,"CODE",{});var $c=l(pn);Nl=a($c,"AutoModel"),$c.forEach(s),Pl=a(H,". Tout ce que nous avons \xE0 faire est de choisir un "),mn=r(H,"EM",{});var qc=l(mn);Ml=a(qc,"checkpoint"),qc.forEach(s),Rl=a(H," appropri\xE9 \xE0 partir duquel charger le mod\xE8le. Heureusement, il existe une biblioth\xE8que appel\xE9e "),fn=r(H,"CODE",{});var Ec=l(fn);Fl=a(Ec,"sentence-transformers"),Ec.forEach(s),Ll=a(H," d\xE9di\xE9e \xE0 la cr\xE9ation d\u2019ench\xE2ssements. Comme d\xE9crit dans la "),kt=r(H,"A",{href:!0,rel:!0});var xc=l(kt);Hl=a(xc,"documentation de la biblioth\xE8que"),xc.forEach(s),zl=a(H,", notre cas d\u2019utilisation est un exemple de "),hn=r(H,"EM",{});var kc=l(hn);Ul=a(kc,"recherche s\xE9mantique asym\xE9trique"),kc.forEach(s),Gl=a(H,". En effet, nous avons une requ\xEAte courte dont nous aimerions trouver la r\xE9ponse dans un document plus long, par exemple un commentaire \xE0 un probl\xE8me. Le "),wt=r(H,"A",{href:!0,rel:!0});var wc=l(wt);Vl=a(wc,"tableau de pr\xE9sentation des mod\xE8les"),wc.forEach(s),Yl=a(H," de la documentation indique que le "),_n=r(H,"EM",{});var yc=l(_n);Bl=a(yc,"checkpoint"),yc.forEach(s),Wl=p(H),gn=r(H,"CODE",{});var jc=l(gn);Jl=a(jc,"multi-qa-mpnet-base-dot-v1"),jc.forEach(s),Ql=a(H," a les meilleures performances pour la recherche s\xE9mantique. Utilisons donc le pour notre application. Nous allons \xE9galement charger le "),vn=r(H,"EM",{});var Dc=l(vn);Xl=a(Dc,"tokenizer"),Dc.forEach(s),Kl=a(H," en utilisant le m\xEAme "),bn=r(H,"EM",{});var Cc=l(bn);Zl=a(Cc,"checkpoint"),Cc.forEach(s),ei=a(H," :"),H.forEach(s),wa=p(e),be.l(e),Yt=p(e),ne=r(e,"P",{});var Ie=l(ne);ti=a(Ie,"Comme nous l\u2019avons mentionn\xE9 pr\xE9c\xE9demment, nous aimerions repr\xE9senter chaque entr\xE9e dans notre corpus de probl\xE8mes GitHub comme un vecteur unique. Nous devons donc regrouper ou faire la moyenne de nos ench\xE2ssements de "),$n=r(Ie,"EM",{});var Tc=l($n);si=a(Tc,"tokens"),Tc.forEach(s),ni=a(Ie," d\u2019une mani\xE8re ou d\u2019une autre. Une approche populaire consiste \xE0 effectuer un "),qn=r(Ie,"EM",{});var Ac=l(qn);ai=a(Ac,"regroupement CLS"),Ac.forEach(s),oi=a(Ie," sur les sorties de notre mod\xE8le, o\xF9 nous collectons simplement le dernier \xE9tat cach\xE9 pour le "),En=r(Ie,"EM",{});var Sc=l(En);ri=a(Sc,"token"),Sc.forEach(s),li=a(Ie," sp\xE9cial "),xn=r(Ie,"CODE",{});var Oc=l(xn);ii=a(Oc,"[CLS]"),Oc.forEach(s),ui=a(Ie,". La fonction suivante fait \xE7a pour nous :"),Ie.forEach(s),ya=p(e),x(yt.$$.fragment,e),ja=p(e),Bt=r(e,"P",{});var Ic=l(Bt);ci=a(Ic,"Ensuite, nous allons cr\xE9er une fonction utile qui va tokeniser une liste de documents, placer les tenseurs dans le GPU, les donner au mod\xE8le et enfin appliquer le regroupement CLS aux sorties :"),Ic.forEach(s),Da=p(e),qe.l(e),Wt=p(e),We=r(e,"P",{});var Ka=l(We);di=a(Ka,"Notez que nous avons converti les ench\xE2ssements en tableaux NumPy. C\u2019est parce que \u{1F917} "),kn=r(Ka,"EM",{});var Nc=l(kn);pi=a(Nc,"Datasets"),Nc.forEach(s),mi=a(Ka," n\xE9cessite ce format lorsque nous essayons de les indexer avec FAISS, ce que nous ferons ensuite."),Ka.forEach(s),Ca=p(e),Le=r(e,"H2",{class:!0});var Za=l(Le);Je=r(Za,"A",{id:!0,class:!0,href:!0});var Pc=l(Je);wn=r(Pc,"SPAN",{});var Mc=l(wn);x(jt.$$.fragment,Mc),Mc.forEach(s),Pc.forEach(s),fi=p(Za),yn=r(Za,"SPAN",{});var Rc=l(yn);hi=a(Rc,"Utilisation de FAISS pour une recherche de similarit\xE9 efficace"),Rc.forEach(s),Za.forEach(s),Ta=p(e),ae=r(e,"P",{});var Ne=l(ae);_i=a(Ne,"Maintenant que nous avons un jeu de donn\xE9es d\u2019incorporations, nous avons besoin d\u2019un moyen de les rechercher. Pour ce faire, nous utiliserons une structure de donn\xE9es sp\xE9ciale dans \u{1F917} "),jn=r(Ne,"EM",{});var Fc=l(jn);gi=a(Fc,"Datasets"),Fc.forEach(s),vi=a(Ne," appel\xE9e "),Dn=r(Ne,"EM",{});var Lc=l(Dn);bi=a(Lc,"FAISS index"),Lc.forEach(s),$i=a(Ne,". "),Dt=r(Ne,"A",{href:!0,rel:!0});var Hc=l(Dt);qi=a(Hc,"FAISS"),Hc.forEach(s),Ei=a(Ne," (abr\xE9viation de "),Cn=r(Ne,"EM",{});var zc=l(Cn);xi=a(zc,"Facebook AI Similarity Search"),zc.forEach(s),ki=a(Ne,") est une biblioth\xE8que qui fournit des algorithmes efficaces pour rechercher et regrouper rapidement des vecteurs d\u2019int\xE9gration."),Ne.forEach(s),Aa=p(e),me=r(e,"P",{});var et=l(me);wi=a(et,"L\u2019id\xE9e de base derri\xE8re FAISS est de cr\xE9er une structure de donn\xE9es sp\xE9ciale appel\xE9e un "),Tn=r(et,"EM",{});var Uc=l(Tn);yi=a(Uc,"index"),Uc.forEach(s),ji=a(et," qui permet de trouver quels plongements sont similaires \xE0 un plongement d\u2019entr\xE9e. Cr\xE9er un index FAISS dans \u{1F917} "),An=r(et,"EM",{});var Gc=l(An);Di=a(Gc,"Datasets"),Gc.forEach(s),Ci=a(et," est simple \u2014 nous utilisons la fonction "),Sn=r(et,"CODE",{});var Vc=l(Sn);Ti=a(Vc,"Dataset.add_faiss_index()"),Vc.forEach(s),Ai=a(et," et sp\xE9cifions quelle colonne de notre jeu de donn\xE9es nous aimerions indexer :"),et.forEach(s),Sa=p(e),x(Ct.$$.fragment,e),Oa=p(e),Qe=r(e,"P",{});var eo=l(Qe);Si=a(eo,"Nous pouvons maintenant effectuer des requ\xEAtes sur cet index en effectuant une recherche des voisins les plus proches avec la fonction "),On=r(eo,"CODE",{});var Yc=l(On);Oi=a(Yc,"Dataset.get_nearest_examples()"),Yc.forEach(s),Ii=a(eo,". Testons cela en ench\xE2ssant d\u2019abord une question comme suit :"),eo.forEach(s),Ia=p(e),xe.l(e),Jt=p(e),Qt=r(e,"P",{});var Bc=l(Qt);Ni=a(Bc,"Tout comme avec les documents, nous avons maintenant un vecteur de 768 dimensions repr\xE9sentant la requ\xEAte. Nous pouvons le comparer \xE0 l\u2019ensemble du corpus pour trouver les ench\xE2ssements les plus similaires :"),Bc.forEach(s),Na=p(e),x(Tt.$$.fragment,e),Pa=p(e),fe=r(e,"P",{});var tt=l(fe);Pi=a(tt,"La fonction "),In=r(tt,"CODE",{});var Wc=l(In);Mi=a(Wc,"Dataset.get_nearest_examples()"),Wc.forEach(s),Ri=a(tt," renvoie un "),Nn=r(tt,"EM",{});var Jc=l(Nn);Fi=a(Jc,"tuple"),Jc.forEach(s),Li=a(tt," de scores qui classent le chevauchement entre la requ\xEAte et le document, et un jeu correspondant d\u2019\xE9chantillons (ici, les 5 meilleures correspondances). Collectons-les dans un "),Pn=r(tt,"CODE",{});var Qc=l(Pn);Hi=a(Qc,"pandas.DataFrame"),Qc.forEach(s),zi=a(tt," afin de pouvoir les trier facilement :"),tt.forEach(s),Ma=p(e),x(At.$$.fragment,e),Ra=p(e),Xt=r(e,"P",{});var Xc=l(Xt);Ui=a(Xc,"Nous pouvons maintenant parcourir les premi\xE8res lignes pour voir dans quelle mesure notre requ\xEAte correspond aux commentaires disponibles :"),Xc.forEach(s),Fa=p(e),x(St.$$.fragment,e),La=p(e),x(Ot.$$.fragment,e),Ha=p(e),Kt=r(e,"P",{});var Kc=l(Kt);Gi=a(Kc,"Pas mal ! Notre deuxi\xE8me r\xE9sultat semble correspondre \xE0 la requ\xEAte."),Kc.forEach(s),za=p(e),x(Xe.$$.fragment,e),this.h()},h(){y(m,"name","hf:doc:metadata"),y(m,"content",JSON.stringify($d)),y($,"id","recherche-smantique-avec-faiss"),y($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),y($,"href","#recherche-smantique-avec-faiss"),y(_,"class","relative group"),y(P,"href","/course/fr/chapter5/5"),y(J,"id","utilisation-des-enchssements-pour-la-recherche-smantique"),y(J,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),y(J,"href","#utilisation-des-enchssements-pour-la-recherche-smantique"),y(W,"class","relative group"),y(Ft,"href","/course/fr/chapter1"),y(st,"class","block dark:hidden"),Zc(st.src,Vi="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter5/semantic-search.svg")||y(st,"src",Vi),y(st,"alt","Recherche s\xE9mantique."),y(nt,"class","hidden dark:block"),Zc(nt.src,Yi="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter5/semantic-search-dark.svg")||y(nt,"src",Yi),y(nt,"alt","Recherche s\xE9mantique."),y(Me,"class","flex justify-center"),y(ze,"id","chargement-et-prparation-du-jeu-de-donnes"),y(ze,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),y(ze,"href","#chargement-et-prparation-du-jeu-de-donnes"),y(Re,"class","relative group"),y(Ht,"href","/course/fr/chapter5/2"),y(pt,"href","https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.explode.html"),y(pt,"rel","nofollow"),to(se,"text-align","right"),y(le,"border","1"),y(le,"class","dataframe"),to(le,"table-layout","fixed"),to(le,"word-wrap","break-word"),to(le,"width","100%"),y(Be,"id","cration-denchssements-pour-les-textes"),y(Be,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),y(Be,"href","#cration-denchssements-pour-les-textes"),y(Fe,"class","relative group"),y(Vt,"href","/course/fr/chapter2"),y(kt,"href","https://www.sbert.net/examples/applications/semantic-search/README.html#symmetric-vs-asymmetric-semantic-search"),y(kt,"rel","nofollow"),y(wt,"href","https://www.sbert.net/docs/pretrained_models.html#model-overview"),y(wt,"rel","nofollow"),y(Je,"id","utilisation-de-faiss-pour-une-recherche-de-similarit-efficace"),y(Je,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),y(Je,"href","#utilisation-de-faiss-pour-une-recherche-de-similarit-efficace"),y(Le,"class","relative group"),y(Dt,"href","https://faiss.ai/"),y(Dt,"rel","nofollow")},m(e,i){t(document.head,m),u(e,q,i),k(f,e,i),u(e,j,i),u(e,_,i),t(_,$),t($,S),k(g,S,null),t(_,C),t(_,D),t(D,z),u(e,O,i),It[T].m(e,i),u(e,N,i),u(e,R,i),t(R,Y),t(R,P),t(P,Q),t(R,B),t(R,U),t(U,L),t(R,V),u(e,c,i),k(h,e,i),u(e,F,i),u(e,W,i),t(W,J),t(J,K),k(re,K,null),t(W,Pe),t(W,He),t(He,ao),u(e,Ln,i),u(e,te,i),t(te,oo),t(te,Ft),t(Ft,ro),t(te,lo),t(te,os),t(os,io),t(te,uo),t(te,rs),t(rs,co),t(te,po),t(te,ls),t(ls,mo),t(te,fo),u(e,Hn,i),u(e,Lt,i),t(Lt,ho),u(e,zn,i),u(e,Me,i),t(Me,st),t(Me,_o),t(Me,nt),u(e,Un,i),u(e,Re,i),t(Re,ze),t(ze,is),k(at,is,null),t(Re,go),t(Re,us),t(us,vo),u(e,Gn,i),u(e,ke,i),t(ke,bo),t(ke,cs),t(cs,$o),t(ke,qo),t(ke,ds),t(ds,Eo),t(ke,xo),u(e,Vn,i),k(ot,e,i),u(e,Yn,i),u(e,we,i),t(we,ko),t(we,ps),t(ps,wo),t(we,yo),t(we,Ht),t(Ht,jo),t(we,Do),u(e,Bn,i),k(rt,e,i),u(e,Wn,i),k(lt,e,i),u(e,Jn,i),u(e,X,i),t(X,Co),t(X,ms),t(ms,To),t(X,Ao),t(X,fs),t(fs,So),t(X,Oo),t(X,hs),t(hs,Io),t(X,No),t(X,_s),t(_s,Po),t(X,Mo),t(X,gs),t(gs,Ro),t(X,Fo),t(X,vs),t(vs,Lo),t(X,Ho),u(e,Qn,i),k(it,e,i),u(e,Xn,i),k(ut,e,i),u(e,Kn,i),u(e,Z,i),t(Z,zo),t(Z,bs),t(bs,Uo),t(Z,Go),t(Z,$s),t($s,Vo),t(Z,Yo),t(Z,qs),t(qs,Bo),t(Z,Wo),t(Z,Es),t(Es,Jo),t(Z,Qo),t(Z,xs),t(xs,Xo),t(Z,Ko),u(e,Zn,i),k(ct,e,i),u(e,ea,i),k(dt,e,i),u(e,ta,i),u(e,ee,i),t(ee,Zo),t(ee,ks),t(ks,er),t(ee,tr),t(ee,ws),t(ws,sr),t(ee,nr),t(ee,ys),t(ys,ar),t(ee,or),t(ee,pt),t(pt,js),t(js,rr),t(ee,lr),t(ee,Ds),t(Ds,ir),t(ee,ur),u(e,sa,i),k(mt,e,i),u(e,na,i),u(e,Ue,i),t(Ue,cr),t(Ue,Cs),t(Cs,dr),t(Ue,pr),u(e,aa,i),k(ft,e,i),u(e,oa,i),k(ht,e,i),u(e,ra,i),u(e,Ge,i),t(Ge,mr),t(Ge,Ts),t(Ts,fr),t(Ge,hr),u(e,la,i),k(_t,e,i),u(e,ia,i),u(e,le,i),t(le,As),t(As,se),t(se,ua),t(se,_r),t(se,Ss),t(Ss,gr),t(se,vr),t(se,Os),t(Os,br),t(se,$r),t(se,Is),t(Is,qr),t(se,Er),t(se,Ns),t(Ns,xr),t(le,kr),t(le,ge),t(ge,ie),t(ie,Ps),t(Ps,wr),t(ie,yr),t(ie,Ms),t(Ms,jr),t(ie,Dr),t(ie,Rs),t(Rs,Cr),t(ie,Tr),t(ie,Fs),t(Fs,Ar),t(ie,Sr),t(ie,Ls),t(Ls,Or),t(ge,Ir),t(ge,ue),t(ue,Hs),t(Hs,Nr),t(ue,Pr),t(ue,zs),t(zs,Mr),t(ue,Rr),t(ue,Us),t(Us,Fr),t(ue,Lr),t(ue,Gs),t(Gs,Hr),t(ue,zr),t(ue,Vs),t(Vs,Ur),t(ge,Gr),t(ge,ce),t(ce,Ys),t(Ys,Vr),t(ce,Yr),t(ce,Bs),t(Bs,Br),t(ce,Wr),t(ce,Ws),t(Ws,Jr),t(ce,Qr),t(ce,Js),t(Js,Xr),t(ce,Kr),t(ce,Qs),t(Qs,Zr),t(ge,el),t(ge,de),t(de,Xs),t(Xs,tl),t(de,sl),t(de,Ks),t(Ks,nl),t(de,al),t(de,Zs),t(Zs,ol),t(de,rl),t(de,en),t(en,ll),t(de,il),t(de,tn),t(tn,ul),u(e,ca,i),u(e,pe,i),t(pe,cl),t(pe,sn),t(sn,dl),t(pe,pl),t(pe,nn),t(nn,ml),t(pe,fl),t(pe,an),t(an,hl),t(pe,_l),u(e,da,i),k(gt,e,i),u(e,pa,i),k(vt,e,i),u(e,ma,i),u(e,zt,i),t(zt,gl),u(e,fa,i),k(Ve,e,i),u(e,ha,i),u(e,Ye,i),t(Ye,vl),t(Ye,on),t(on,bl),t(Ye,$l),u(e,_a,i),k(bt,e,i),u(e,ga,i),u(e,Ut,i),t(Ut,ql),u(e,va,i),k($t,e,i),u(e,ba,i),k(qt,e,i),u(e,$a,i),u(e,ye,i),t(ye,El),t(ye,rn),t(rn,xl),t(ye,kl),t(ye,ln),t(ln,wl),t(ye,yl),u(e,qa,i),k(Et,e,i),u(e,Ea,i),u(e,Gt,i),t(Gt,jl),u(e,xa,i),u(e,Fe,i),t(Fe,Be),t(Be,un),k(xt,un,null),t(Fe,Dl),t(Fe,cn),t(cn,Cl),u(e,ka,i),u(e,M,i),t(M,Tl),t(M,Vt),t(Vt,Al),t(M,Sl),t(M,dn),t(dn,Ol),t(M,Il),t(M,pn),t(pn,Nl),t(M,Pl),t(M,mn),t(mn,Ml),t(M,Rl),t(M,fn),t(fn,Fl),t(M,Ll),t(M,kt),t(kt,Hl),t(M,zl),t(M,hn),t(hn,Ul),t(M,Gl),t(M,wt),t(wt,Vl),t(M,Yl),t(M,_n),t(_n,Bl),t(M,Wl),t(M,gn),t(gn,Jl),t(M,Ql),t(M,vn),t(vn,Xl),t(M,Kl),t(M,bn),t(bn,Zl),t(M,ei),u(e,wa,i),Nt[ve].m(e,i),u(e,Yt,i),u(e,ne,i),t(ne,ti),t(ne,$n),t($n,si),t(ne,ni),t(ne,qn),t(qn,ai),t(ne,oi),t(ne,En),t(En,ri),t(ne,li),t(ne,xn),t(xn,ii),t(ne,ui),u(e,ya,i),k(yt,e,i),u(e,ja,i),u(e,Bt,i),t(Bt,ci),u(e,Da,i),Pt[$e].m(e,i),u(e,Wt,i),u(e,We,i),t(We,di),t(We,kn),t(kn,pi),t(We,mi),u(e,Ca,i),u(e,Le,i),t(Le,Je),t(Je,wn),k(jt,wn,null),t(Le,fi),t(Le,yn),t(yn,hi),u(e,Ta,i),u(e,ae,i),t(ae,_i),t(ae,jn),t(jn,gi),t(ae,vi),t(ae,Dn),t(Dn,bi),t(ae,$i),t(ae,Dt),t(Dt,qi),t(ae,Ei),t(ae,Cn),t(Cn,xi),t(ae,ki),u(e,Aa,i),u(e,me,i),t(me,wi),t(me,Tn),t(Tn,yi),t(me,ji),t(me,An),t(An,Di),t(me,Ci),t(me,Sn),t(Sn,Ti),t(me,Ai),u(e,Sa,i),k(Ct,e,i),u(e,Oa,i),u(e,Qe,i),t(Qe,Si),t(Qe,On),t(On,Oi),t(Qe,Ii),u(e,Ia,i),Mt[Ee].m(e,i),u(e,Jt,i),u(e,Qt,i),t(Qt,Ni),u(e,Na,i),k(Tt,e,i),u(e,Pa,i),u(e,fe,i),t(fe,Pi),t(fe,In),t(In,Mi),t(fe,Ri),t(fe,Nn),t(Nn,Fi),t(fe,Li),t(fe,Pn),t(Pn,Hi),t(fe,zi),u(e,Ma,i),k(At,e,i),u(e,Ra,i),u(e,Xt,i),t(Xt,Ui),u(e,Fa,i),k(St,e,i),u(e,La,i),k(Ot,e,i),u(e,Ha,i),u(e,Kt,i),t(Kt,Gi),u(e,za,i),k(Xe,e,i),Ua=!0},p(e,[i]){const Rt={};i&1&&(Rt.fw=e[0]),f.$set(Rt);let Zt=T;T=Wi(e),T!==Zt&&(no(),v(It[Zt],1,1,()=>{It[Zt]=null}),so(),I=It[T],I||(I=It[T]=Bi[T](e),I.c()),b(I,1),I.m(N.parentNode,N));const Mn={};i&2&&(Mn.$$scope={dirty:i,ctx:e}),Ve.$set(Mn);let es=ve;ve=Qi(e),ve!==es&&(no(),v(Nt[es],1,1,()=>{Nt[es]=null}),so(),be=Nt[ve],be||(be=Nt[ve]=Ji[ve](e),be.c()),b(be,1),be.m(Yt.parentNode,Yt));let je=$e;$e=Ki(e),$e!==je&&(no(),v(Pt[je],1,1,()=>{Pt[je]=null}),so(),qe=Pt[$e],qe||(qe=Pt[$e]=Xi[$e](e),qe.c()),b(qe,1),qe.m(Wt.parentNode,Wt));let ts=Ee;Ee=eu(e),Ee!==ts&&(no(),v(Mt[ts],1,1,()=>{Mt[ts]=null}),so(),xe=Mt[Ee],xe||(xe=Mt[Ee]=Zi[Ee](e),xe.c()),b(xe,1),xe.m(Jt.parentNode,Jt));const Rn={};i&2&&(Rn.$$scope={dirty:i,ctx:e}),Xe.$set(Rn)},i(e){Ua||(b(f.$$.fragment,e),b(g.$$.fragment,e),b(I),b(h.$$.fragment,e),b(re.$$.fragment,e),b(at.$$.fragment,e),b(ot.$$.fragment,e),b(rt.$$.fragment,e),b(lt.$$.fragment,e),b(it.$$.fragment,e),b(ut.$$.fragment,e),b(ct.$$.fragment,e),b(dt.$$.fragment,e),b(mt.$$.fragment,e),b(ft.$$.fragment,e),b(ht.$$.fragment,e),b(_t.$$.fragment,e),b(gt.$$.fragment,e),b(vt.$$.fragment,e),b(Ve.$$.fragment,e),b(bt.$$.fragment,e),b($t.$$.fragment,e),b(qt.$$.fragment,e),b(Et.$$.fragment,e),b(xt.$$.fragment,e),b(be),b(yt.$$.fragment,e),b(qe),b(jt.$$.fragment,e),b(Ct.$$.fragment,e),b(xe),b(Tt.$$.fragment,e),b(At.$$.fragment,e),b(St.$$.fragment,e),b(Ot.$$.fragment,e),b(Xe.$$.fragment,e),Ua=!0)},o(e){v(f.$$.fragment,e),v(g.$$.fragment,e),v(I),v(h.$$.fragment,e),v(re.$$.fragment,e),v(at.$$.fragment,e),v(ot.$$.fragment,e),v(rt.$$.fragment,e),v(lt.$$.fragment,e),v(it.$$.fragment,e),v(ut.$$.fragment,e),v(ct.$$.fragment,e),v(dt.$$.fragment,e),v(mt.$$.fragment,e),v(ft.$$.fragment,e),v(ht.$$.fragment,e),v(_t.$$.fragment,e),v(gt.$$.fragment,e),v(vt.$$.fragment,e),v(Ve.$$.fragment,e),v(bt.$$.fragment,e),v($t.$$.fragment,e),v(qt.$$.fragment,e),v(Et.$$.fragment,e),v(xt.$$.fragment,e),v(be),v(yt.$$.fragment,e),v(qe),v(jt.$$.fragment,e),v(Ct.$$.fragment,e),v(xe),v(Tt.$$.fragment,e),v(At.$$.fragment,e),v(St.$$.fragment,e),v(Ot.$$.fragment,e),v(Xe.$$.fragment,e),Ua=!1},d(e){s(m),e&&s(q),w(f,e),e&&s(j),e&&s(_),w(g),e&&s(O),It[T].d(e),e&&s(N),e&&s(R),e&&s(c),w(h,e),e&&s(F),e&&s(W),w(re),e&&s(Ln),e&&s(te),e&&s(Hn),e&&s(Lt),e&&s(zn),e&&s(Me),e&&s(Un),e&&s(Re),w(at),e&&s(Gn),e&&s(ke),e&&s(Vn),w(ot,e),e&&s(Yn),e&&s(we),e&&s(Bn),w(rt,e),e&&s(Wn),w(lt,e),e&&s(Jn),e&&s(X),e&&s(Qn),w(it,e),e&&s(Xn),w(ut,e),e&&s(Kn),e&&s(Z),e&&s(Zn),w(ct,e),e&&s(ea),w(dt,e),e&&s(ta),e&&s(ee),e&&s(sa),w(mt,e),e&&s(na),e&&s(Ue),e&&s(aa),w(ft,e),e&&s(oa),w(ht,e),e&&s(ra),e&&s(Ge),e&&s(la),w(_t,e),e&&s(ia),e&&s(le),e&&s(ca),e&&s(pe),e&&s(da),w(gt,e),e&&s(pa),w(vt,e),e&&s(ma),e&&s(zt),e&&s(fa),w(Ve,e),e&&s(ha),e&&s(Ye),e&&s(_a),w(bt,e),e&&s(ga),e&&s(Ut),e&&s(va),w($t,e),e&&s(ba),w(qt,e),e&&s($a),e&&s(ye),e&&s(qa),w(Et,e),e&&s(Ea),e&&s(Gt),e&&s(xa),e&&s(Fe),w(xt),e&&s(ka),e&&s(M),e&&s(wa),Nt[ve].d(e),e&&s(Yt),e&&s(ne),e&&s(ya),w(yt,e),e&&s(ja),e&&s(Bt),e&&s(Da),Pt[$e].d(e),e&&s(Wt),e&&s(We),e&&s(Ca),e&&s(Le),w(jt),e&&s(Ta),e&&s(ae),e&&s(Aa),e&&s(me),e&&s(Sa),w(Ct,e),e&&s(Oa),e&&s(Qe),e&&s(Ia),Mt[Ee].d(e),e&&s(Jt),e&&s(Qt),e&&s(Na),w(Tt,e),e&&s(Pa),e&&s(fe),e&&s(Ma),w(At,e),e&&s(Ra),e&&s(Xt),e&&s(Fa),w(St,e),e&&s(La),w(Ot,e),e&&s(Ha),e&&s(Kt),e&&s(za),w(Xe,e)}}}const $d={local:"recherche-smantique-avec-faiss",sections:[{local:"utilisation-des-enchssements-pour-la-recherche-smantique",title:"Utilisation des ench\xE2ssements pour la recherche s\xE9mantique"},{local:"chargement-et-prparation-du-jeu-de-donnes",title:"Chargement et pr\xE9paration du jeu de donn\xE9es"},{local:"cration-denchssements-pour-les-textes",title:"Cr\xE9ation d\u2019ench\xE2ssements pour les textes"},{local:"utilisation-de-faiss-pour-une-recherche-de-similarit-efficace",title:"Utilisation de FAISS pour une recherche de similarit\xE9 efficace"}],title:"Recherche s\xE9mantique avec FAISS"};function qd(G,m,q){let f="pt";return rd(()=>{const j=new URLSearchParams(window.location.search);q(0,f=j.get("fw")||"pt")}),[f]}class Cd extends sd{constructor(m){super();nd(this,m,qd,bd,ad,{})}}export{Cd as default,$d as metadata};
