import{S as cp,i as mp,s as fp,e as s,k as i,w as g,t as n,M as up,c as a,d as r,m as d,a as o,x as v,h as l,b as m,G as t,g as c,y as $,q as b,o as x,B as E,v as hp,L as vi}from"../../chunks/vendor-hf-doc-builder.js";import{T as _i}from"../../chunks/Tip-hf-doc-builder.js";import{D as k}from"../../chunks/Docstring-hf-doc-builder.js";import{C as Ds}from"../../chunks/CodeBlock-hf-doc-builder.js";import{I as Qe}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{E as gi}from"../../chunks/ExampleCodeBlock-hf-doc-builder.js";function _p(z){let u,y,_,h,P,f,w,C,T,A,j;return{c(){u=s("p"),y=n(`This class method is simply calling the feature extractor
`),_=s("a"),h=n("from_pretrained()"),P=n(`, image processor
`),f=s("a"),w=n("ImageProcessingMixin"),C=n(` and the tokenizer
`),T=s("code"),A=n("~tokenization_utils_base.PreTrainedTokenizer.from_pretrained"),j=n(` methods. Please refer to the docstrings of the
methods above for more information.`),this.h()},l(O){u=a(O,"P",{});var N=o(u);y=l(N,`This class method is simply calling the feature extractor
`),_=a(N,"A",{href:!0});var F=o(_);h=l(F,"from_pretrained()"),F.forEach(r),P=l(N,`, image processor
`),f=a(N,"A",{href:!0});var Q=o(f);w=l(Q,"ImageProcessingMixin"),Q.forEach(r),C=l(N,` and the tokenizer
`),T=a(N,"CODE",{});var Qt=o(T);A=l(Qt,"~tokenization_utils_base.PreTrainedTokenizer.from_pretrained"),Qt.forEach(r),j=l(N,` methods. Please refer to the docstrings of the
methods above for more information.`),N.forEach(r),this.h()},h(){m(_,"href","/docs/transformers/main/en/main_classes/feature_extractor#transformers.FeatureExtractionMixin.from_pretrained"),m(f,"href","/docs/transformers/main/en/internal/image_processing_utils#transformers.ImageProcessingMixin")},m(O,N){c(O,u,N),t(u,y),t(u,_),t(_,h),t(u,P),t(u,f),t(f,w),t(u,C),t(u,T),t(T,A),t(u,j)},d(O){O&&r(u)}}}function gp(z){let u,y,_,h,P;return h=new Ds({props:{code:`from transformers import AutoProcessor

processor = AutoProcessor.from_pretrained("bert-base-cased")

# Push the processor to your namespace with the name "my-finetuned-bert".
processor.push_to_hub("my-finetuned-bert")

# Push the processor to an organization with the name "my-finetuned-bert".
processor.push_to_hub("huggingface/my-finetuned-bert")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoProcessor

processor = AutoProcessor.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)

<span class="hljs-comment"># Push the processor to your namespace with the name &quot;my-finetuned-bert&quot;.</span>
processor.push_to_hub(<span class="hljs-string">&quot;my-finetuned-bert&quot;</span>)

<span class="hljs-comment"># Push the processor to an organization with the name &quot;my-finetuned-bert&quot;.</span>
processor.push_to_hub(<span class="hljs-string">&quot;huggingface/my-finetuned-bert&quot;</span>)`}}),{c(){u=s("p"),y=n("Examples:"),_=i(),g(h.$$.fragment)},l(f){u=a(f,"P",{});var w=o(u);y=l(w,"Examples:"),w.forEach(r),_=d(f),v(h.$$.fragment,f)},m(f,w){c(f,u,w),t(u,y),c(f,_,w),$(h,f,w),P=!0},p:vi,i(f){P||(b(h.$$.fragment,f),P=!0)},o(f){x(h.$$.fragment,f),P=!1},d(f){f&&r(u),f&&r(_),E(h,f)}}}function vp(z){let u,y;return{c(){u=s("p"),y=n("This API is experimental and may have some slight breaking changes in the next releases.")},l(_){u=a(_,"P",{});var h=o(u);y=l(h,"This API is experimental and may have some slight breaking changes in the next releases."),h.forEach(r)},m(_,h){c(_,u,h),t(u,y)},d(_){_&&r(u)}}}function $p(z){let u,y,_,h,P,f,w,C;return{c(){u=s("p"),y=n("This class method is simply calling "),_=s("a"),h=n("save_pretrained()"),P=n(` and
`),f=s("a"),w=n("save_pretrained()"),C=n(`. Please refer to the docstrings of the
methods above for more information.`),this.h()},l(T){u=a(T,"P",{});var A=o(u);y=l(A,"This class method is simply calling "),_=a(A,"A",{href:!0});var j=o(_);h=l(j,"save_pretrained()"),j.forEach(r),P=l(A,` and
`),f=a(A,"A",{href:!0});var O=o(f);w=l(O,"save_pretrained()"),O.forEach(r),C=l(A,`. Please refer to the docstrings of the
methods above for more information.`),A.forEach(r),this.h()},h(){m(_,"href","/docs/transformers/main/en/main_classes/feature_extractor#transformers.FeatureExtractionMixin.save_pretrained"),m(f,"href","/docs/transformers/main/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.save_pretrained")},m(T,A){c(T,u,A),t(u,y),t(u,_),t(_,h),t(u,P),t(u,f),t(f,w),t(u,C)},d(T){T&&r(u)}}}function bp(z){let u,y,_,h,P;return h=new Ds({props:{code:`import tensorflow_datasets as tfds

dataset = tfds.load("squad")

training_examples = get_examples_from_dataset(dataset, evaluate=False)
evaluation_examples = get_examples_from_dataset(dataset, evaluate=True)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> tensorflow_datasets <span class="hljs-keyword">as</span> tfds

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = tfds.load(<span class="hljs-string">&quot;squad&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>training_examples = get_examples_from_dataset(dataset, evaluate=<span class="hljs-literal">False</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>evaluation_examples = get_examples_from_dataset(dataset, evaluate=<span class="hljs-literal">True</span>)`}}),{c(){u=s("p"),y=n("Examples:"),_=i(),g(h.$$.fragment)},l(f){u=a(f,"P",{});var w=o(u);y=l(w,"Examples:"),w.forEach(r),_=d(f),v(h.$$.fragment,f)},m(f,w){c(f,u,w),t(u,y),c(f,_,w),$(h,f,w),P=!0},p:vi,i(f){P||(b(h.$$.fragment,f),P=!0)},o(f){x(h.$$.fragment,f),P=!1},d(f){f&&r(u),f&&r(_),E(h,f)}}}function xp(z){let u,y,_,h,P;return h=new Ds({props:{code:`processor = SquadV2Processor()
examples = processor.get_dev_examples(data_dir)

features = squad_convert_examples_to_features(
    examples=examples,
    tokenizer=tokenizer,
    max_seq_length=args.max_seq_length,
    doc_stride=args.doc_stride,
    max_query_length=args.max_query_length,
    is_training=not evaluate,
)`,highlighted:`processor = SquadV2Processor()
examples = processor.get_dev_examples(data_dir)

features = squad_convert_examples_to_features(
    examples=examples,
    tokenizer=tokenizer,
    max_seq_length=args.max_seq_length,
    doc_stride=args.doc_stride,
    max_query_length=args.max_query_length,
    is_training=<span class="hljs-keyword">not</span> evaluate,
)`}}),{c(){u=s("p"),y=n("Example:"),_=i(),g(h.$$.fragment)},l(f){u=a(f,"P",{});var w=o(u);y=l(w,"Example:"),w.forEach(r),_=d(f),v(h.$$.fragment,f)},m(f,w){c(f,u,w),t(u,y),c(f,_,w),$(h,f,w),P=!0},p:vi,i(f){P||(b(h.$$.fragment,f),P=!0)},o(f){x(h.$$.fragment,f),P=!1},d(f){f&&r(u),f&&r(_),E(h,f)}}}function Ep(z){let u,y,_,h,P,f,w,C,T,A,j,O,N,F,Q,Qt,Ut,Za,eo,Ft,to,ro,so,$r,ao,Ts,te,pe,br,Ue,oo,xr,no,Ns,Gt,lo,Ms,Rt,io,zs,L,Fe,po,Er,co,mo,X,Ge,fo,wr,uo,ho,ce,_o,B,Re,go,He,vo,yr,$o,bo,xo,me,Eo,J,We,wo,Xe,yo,Pr,Po,qo,ko,fe,Io,K,Be,So,Je,Ao,Ht,Lo,Do,To,ue,Vs,re,he,qr,Ke,No,kr,Mo,Cs,V,zo,Wt,Vo,Co,Xt,jo,Oo,Bt,Qo,Uo,Jt,Fo,Go,js,I,Ye,Ro,Ir,Ho,Wo,_e,Ze,Xo,et,Bo,Kt,Jo,Ko,Yo,ge,tt,Zo,Sr,en,tn,ve,rt,rn,Ar,sn,an,$e,st,on,at,nn,Yt,ln,dn,pn,be,ot,cn,nt,mn,Zt,fn,un,hn,xe,lt,_n,Lr,gn,Os,G,it,vn,Dr,$n,bn,Ee,dt,xn,Tr,En,Qs,R,pt,wn,Nr,yn,Pn,we,ct,qn,Mr,kn,Us,se,ye,zr,mt,In,Vr,Sn,Fs,Pe,ft,An,Ln,ut,Dn,Gs,er,Tn,Rs,tr,Nn,Hs,q,Cr,jr,Mn,zn,Or,Qr,Vn,Cn,Ur,Fr,jn,On,Gr,Rr,Qn,Un,Hr,Wr,Fn,Gn,Xr,Br,Rn,Hn,Jr,Kr,Wn,Xn,Yr,Zr,Bn,Jn,es,ts,Kn,Ws,qe,Yn,rr,Zn,el,Xs,ae,ht,tl,sr,rl,rs,sl,Bs,oe,ke,ss,_t,al,as,ol,Js,ne,gt,nl,ll,vt,os,il,dl,Ks,$t,pl,bt,cl,Ys,ar,ml,Zs,or,ns,ls,fl,ea,nr,ul,ta,Ie,hl,xt,_l,gl,ra,le,Se,is,Et,vl,ds,$l,sa,H,wt,bl,xl,yt,El,wl,Pt,yl,Pl,aa,lr,ql,oa,ie,Ae,ps,qt,kl,cs,Il,na,ir,Sl,la,Le,ms,fs,Al,Ll,us,hs,Dl,ia,kt,Tl,_s,Nl,da,M,It,Ml,gs,zl,Vl,De,St,Cl,vs,jl,Ol,Y,At,Ql,Lt,Ul,$s,Fl,Gl,Rl,Te,Hl,Ne,Dt,Wl,bs,Xl,pa,Me,Bl,xs,Jl,Kl,ca,W,Tt,Yl,Es,Zl,ei,ze,ma,Ve,ti,ws,ri,si,fa,de,Ce,ys,Nt,ai,Ps,oi,ua,dr,ni,ha,Mt,_a,je,li,qs,ii,di,ga,zt,va,Oe,pi,Vt,ci,mi,$a;return f=new Qe({}),Ue=new Qe({}),Fe=new k({props:{name:"class transformers.ProcessorMixin",anchor:"transformers.ProcessorMixin",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/processing_utils.py#L44"}}),Ge=new k({props:{name:"from_pretrained",anchor:"transformers.ProcessorMixin.from_pretrained",parameters:[{name:"pretrained_model_name_or_path",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.ProcessorMixin.from_pretrained.pretrained_model_name_or_path",description:`<strong>pretrained_model_name_or_path</strong> (<code>str</code> or <code>os.PathLike</code>) &#x2014;
This can be either:</p>
<ul>
<li>a string, the <em>model id</em> of a pretrained feature_extractor hosted inside a model repo on
huggingface.co. Valid model ids can be located at the root-level, like <code>bert-base-uncased</code>, or
namespaced under a user or organization name, like <code>dbmdz/bert-base-german-cased</code>.</li>
<li>a path to a <em>directory</em> containing a feature extractor file saved using the
<a href="/docs/transformers/main/en/main_classes/feature_extractor#transformers.FeatureExtractionMixin.save_pretrained">save_pretrained()</a> method, e.g., <code>./my_model_directory/</code>.</li>
<li>a path or url to a saved feature extractor JSON <em>file</em>, e.g.,
<code>./my_model_directory/preprocessor_config.json</code>.
**kwargs &#x2014;
Additional keyword arguments passed along to both
<a href="/docs/transformers/main/en/main_classes/feature_extractor#transformers.FeatureExtractionMixin.from_pretrained">from_pretrained()</a> and
<code>~tokenization_utils_base.PreTrainedTokenizer.from_pretrained</code>.</li>
</ul>`,name:"pretrained_model_name_or_path"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/processing_utils.py#L157"}}),ce=new _i({props:{$$slots:{default:[_p]},$$scope:{ctx:z}}}),Re=new k({props:{name:"push_to_hub",anchor:"transformers.ProcessorMixin.push_to_hub",parameters:[{name:"repo_id",val:": str"},{name:"use_temp_dir",val:": typing.Optional[bool] = None"},{name:"commit_message",val:": typing.Optional[str] = None"},{name:"private",val:": typing.Optional[bool] = None"},{name:"use_auth_token",val:": typing.Union[bool, str, NoneType] = None"},{name:"max_shard_size",val:": typing.Union[int, str, NoneType] = '10GB'"},{name:"create_pr",val:": bool = False"},{name:"**deprecated_kwargs",val:""}],parametersDescription:[{anchor:"transformers.ProcessorMixin.push_to_hub.repo_id",description:`<strong>repo_id</strong> (<code>str</code>) &#x2014;
The name of the repository you want to push your processor to. It should contain your organization name
when pushing to a given organization.`,name:"repo_id"},{anchor:"transformers.ProcessorMixin.push_to_hub.use_temp_dir",description:`<strong>use_temp_dir</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to use a temporary directory to store the files saved before they are pushed to the Hub.
Will default to <code>True</code> if there is no directory named like <code>repo_id</code>, <code>False</code> otherwise.`,name:"use_temp_dir"},{anchor:"transformers.ProcessorMixin.push_to_hub.commit_message",description:`<strong>commit_message</strong> (<code>str</code>, <em>optional</em>) &#x2014;
Message to commit while pushing. Will default to <code>&quot;Upload processor&quot;</code>.`,name:"commit_message"},{anchor:"transformers.ProcessorMixin.push_to_hub.private",description:`<strong>private</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not the repository created should be private.`,name:"private"},{anchor:"transformers.ProcessorMixin.push_to_hub.use_auth_token",description:`<strong>use_auth_token</strong> (<code>bool</code> or <code>str</code>, <em>optional</em>) &#x2014;
The token to use as HTTP bearer authorization for remote files. If <code>True</code>, will use the token generated
when running <code>huggingface-cli login</code> (stored in <code>~/.huggingface</code>). Will default to <code>True</code> if <code>repo_url</code>
is not specified.`,name:"use_auth_token"},{anchor:"transformers.ProcessorMixin.push_to_hub.max_shard_size",description:`<strong>max_shard_size</strong> (<code>int</code> or <code>str</code>, <em>optional</em>, defaults to <code>&quot;10GB&quot;</code>) &#x2014;
Only applicable for models. The maximum size for a checkpoint before being sharded. Checkpoints shard
will then be each of size lower than this size. If expressed as a string, needs to be digits followed
by a unit (like <code>&quot;5MB&quot;</code>).`,name:"max_shard_size"},{anchor:"transformers.ProcessorMixin.push_to_hub.create_pr",description:`<strong>create_pr</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to create a PR with the uploaded files or directly commit.`,name:"create_pr"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/utils/hub.py#L703"}}),me=new gi({props:{anchor:"transformers.ProcessorMixin.push_to_hub.example",$$slots:{default:[gp]},$$scope:{ctx:z}}}),We=new k({props:{name:"register_for_auto_class",anchor:"transformers.ProcessorMixin.register_for_auto_class",parameters:[{name:"auto_class",val:" = 'AutoProcessor'"}],parametersDescription:[{anchor:"transformers.ProcessorMixin.register_for_auto_class.auto_class",description:`<strong>auto_class</strong> (<code>str</code> or <code>type</code>, <em>optional</em>, defaults to <code>&quot;AutoProcessor&quot;</code>) &#x2014;
The auto class to register this new feature extractor with.`,name:"auto_class"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/processing_utils.py#L191"}}),fe=new _i({props:{warning:!0,$$slots:{default:[vp]},$$scope:{ctx:z}}}),Be=new k({props:{name:"save_pretrained",anchor:"transformers.ProcessorMixin.save_pretrained",parameters:[{name:"save_directory",val:""},{name:"push_to_hub",val:": bool = False"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.ProcessorMixin.save_pretrained.save_directory",description:`<strong>save_directory</strong> (<code>str</code> or <code>os.PathLike</code>) &#x2014;
Directory where the feature extractor JSON file and the tokenizer files will be saved (directory will
be created if it does not exist).`,name:"save_directory"},{anchor:"transformers.ProcessorMixin.save_pretrained.push_to_hub",description:`<strong>push_to_hub</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to push your model to the Hugging Face model hub after saving it. You can specify the
repository you want to push to with <code>repo_id</code> (will default to the name of <code>save_directory</code> in your
namespace).
kwargs &#x2014;
Additional key word arguments passed along to the <a href="/docs/transformers/main/en/internal/image_processing_utils#transformers.ImageProcessingMixin.push_to_hub">push_to_hub()</a> method.`,name:"push_to_hub"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/processing_utils.py#L95"}}),ue=new _i({props:{$$slots:{default:[$p]},$$scope:{ctx:z}}}),Ke=new Qe({}),Ye=new k({props:{name:"class transformers.DataProcessor",anchor:"transformers.DataProcessor",parameters:[],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/processors/utils.py#L80"}}),Ze=new k({props:{name:"get_dev_examples",anchor:"transformers.DataProcessor.get_dev_examples",parameters:[{name:"data_dir",val:""}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/processors/utils.py#L97"}}),tt=new k({props:{name:"get_example_from_tensor_dict",anchor:"transformers.DataProcessor.get_example_from_tensor_dict",parameters:[{name:"tensor_dict",val:""}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/processors/utils.py#L83"}}),rt=new k({props:{name:"get_labels",anchor:"transformers.DataProcessor.get_labels",parameters:[],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/processors/utils.py#L105"}}),st=new k({props:{name:"get_test_examples",anchor:"transformers.DataProcessor.get_test_examples",parameters:[{name:"data_dir",val:""}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/processors/utils.py#L101"}}),ot=new k({props:{name:"get_train_examples",anchor:"transformers.DataProcessor.get_train_examples",parameters:[{name:"data_dir",val:""}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/processors/utils.py#L93"}}),lt=new k({props:{name:"tfds_map",anchor:"transformers.DataProcessor.tfds_map",parameters:[{name:"example",val:""}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/processors/utils.py#L109"}}),it=new k({props:{name:"class transformers.InputExample",anchor:"transformers.InputExample",parameters:[{name:"guid",val:": str"},{name:"text_a",val:": str"},{name:"text_b",val:": typing.Optional[str] = None"},{name:"label",val:": typing.Optional[str] = None"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/processors/utils.py#L30"}}),dt=new k({props:{name:"to_json_string",anchor:"transformers.InputExample.to_json_string",parameters:[],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/processors/utils.py#L49"}}),pt=new k({props:{name:"class transformers.InputFeatures",anchor:"transformers.InputFeatures",parameters:[{name:"input_ids",val:": typing.List[int]"},{name:"attention_mask",val:": typing.Optional[typing.List[int]] = None"},{name:"token_type_ids",val:": typing.Optional[typing.List[int]] = None"},{name:"label",val:": typing.Union[int, float, NoneType] = None"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/processors/utils.py#L55"}}),ct=new k({props:{name:"to_json_string",anchor:"transformers.InputFeatures.to_json_string",parameters:[],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/processors/utils.py#L75"}}),mt=new Qe({}),ht=new k({props:{name:"transformers.glue_convert_examples_to_features",anchor:"transformers.glue_convert_examples_to_features",parameters:[{name:"examples",val:": typing.Union[typing.List[transformers.data.processors.utils.InputExample], ForwardRef('tf.data.Dataset')]"},{name:"tokenizer",val:": PreTrainedTokenizer"},{name:"max_length",val:": typing.Optional[int] = None"},{name:"task",val:" = None"},{name:"label_list",val:" = None"},{name:"output_mode",val:" = None"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/processors/glue.py#L41",returnDescription:`
<p>If the <code>examples</code> input is a <code>tf.data.Dataset</code>, will return a <code>tf.data.Dataset</code> containing the task-specific
features. If the input is a list of <code>InputExamples</code>, will return a list of task-specific <code>InputFeatures</code> which
can be fed to the model.</p>
`}}),_t=new Qe({}),Et=new Qe({}),qt=new Qe({}),It=new k({props:{name:"class transformers.data.processors.squad.SquadProcessor",anchor:"transformers.data.processors.squad.SquadProcessor",parameters:[],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/processors/squad.py#L542"}}),St=new k({props:{name:"get_dev_examples",anchor:"transformers.data.processors.squad.SquadProcessor.get_dev_examples",parameters:[{name:"data_dir",val:""},{name:"filename",val:" = None"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/processors/squad.py#L630"}}),At=new k({props:{name:"get_examples_from_dataset",anchor:"transformers.data.processors.squad.SquadProcessor.get_examples_from_dataset",parameters:[{name:"dataset",val:""},{name:"evaluate",val:" = False"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/processors/squad.py#L575",returnDescription:`
<p>List of SquadExample</p>
`}}),Te=new gi({props:{anchor:"transformers.data.processors.squad.SquadProcessor.get_examples_from_dataset.example",$$slots:{default:[bp]},$$scope:{ctx:z}}}),Dt=new k({props:{name:"get_train_examples",anchor:"transformers.data.processors.squad.SquadProcessor.get_train_examples",parameters:[{name:"data_dir",val:""},{name:"filename",val:" = None"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/processors/squad.py#L608"}}),Tt=new k({props:{name:"transformers.squad_convert_examples_to_features",anchor:"transformers.squad_convert_examples_to_features",parameters:[{name:"examples",val:""},{name:"tokenizer",val:""},{name:"max_seq_length",val:""},{name:"doc_stride",val:""},{name:"max_query_length",val:""},{name:"is_training",val:""},{name:"padding_strategy",val:" = 'max_length'"},{name:"return_dataset",val:" = False"},{name:"threads",val:" = 1"},{name:"tqdm_enabled",val:" = True"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/processors/squad.py#L317",returnDescription:`
<p>list of <code>SquadFeatures</code></p>
`}}),ze=new gi({props:{anchor:"transformers.squad_convert_examples_to_features.example",$$slots:{default:[xp]},$$scope:{ctx:z}}}),Nt=new Qe({}),Mt=new Ds({props:{code:`# Loading a V2 processor
processor = SquadV2Processor()
examples = processor.get_dev_examples(squad_v2_data_dir)

# Loading a V1 processor
processor = SquadV1Processor()
examples = processor.get_dev_examples(squad_v1_data_dir)

features = squad_convert_examples_to_features(
    examples=examples,
    tokenizer=tokenizer,
    max_seq_length=max_seq_length,
    doc_stride=args.doc_stride,
    max_query_length=max_query_length,
    is_training=not evaluate,
)`,highlighted:`<span class="hljs-comment"># Loading a V2 processor</span>
processor = SquadV2Processor()
examples = processor.get_dev_examples(squad_v2_data_dir)

<span class="hljs-comment"># Loading a V1 processor</span>
processor = SquadV1Processor()
examples = processor.get_dev_examples(squad_v1_data_dir)

features = squad_convert_examples_to_features(
    examples=examples,
    tokenizer=tokenizer,
    max_seq_length=max_seq_length,
    doc_stride=args.doc_stride,
    max_query_length=max_query_length,
    is_training=<span class="hljs-keyword">not</span> evaluate,
)`}}),zt=new Ds({props:{code:`# tensorflow_datasets only handle Squad V1.
tfds_examples = tfds.load("squad")
examples = SquadV1Processor().get_examples_from_dataset(tfds_examples, evaluate=evaluate)

features = squad_convert_examples_to_features(
    examples=examples,
    tokenizer=tokenizer,
    max_seq_length=max_seq_length,
    doc_stride=args.doc_stride,
    max_query_length=max_query_length,
    is_training=not evaluate,
)`,highlighted:`<span class="hljs-comment"># tensorflow_datasets only handle Squad V1.</span>
tfds_examples = tfds.load(<span class="hljs-string">&quot;squad&quot;</span>)
examples = SquadV1Processor().get_examples_from_dataset(tfds_examples, evaluate=evaluate)

features = squad_convert_examples_to_features(
    examples=examples,
    tokenizer=tokenizer,
    max_seq_length=max_seq_length,
    doc_stride=args.doc_stride,
    max_query_length=max_query_length,
    is_training=<span class="hljs-keyword">not</span> evaluate,
)`}}),{c(){u=s("meta"),y=i(),_=s("h1"),h=s("a"),P=s("span"),g(f.$$.fragment),w=i(),C=s("span"),T=n("Processors"),A=i(),j=s("p"),O=n("Processors can mean two different things in the Transformers library:"),N=i(),F=s("ul"),Q=s("li"),Qt=n("the objects that pre-process inputs for multi-modal models such as "),Ut=s("a"),Za=n("Wav2Vec2"),eo=n(` (speech and text)
or `),Ft=s("a"),to=n("CLIP"),ro=n(" (text and vision)"),so=i(),$r=s("li"),ao=n("deprecated objects that were used in older versions of the library to preprocess data for GLUE or SQUAD."),Ts=i(),te=s("h2"),pe=s("a"),br=s("span"),g(Ue.$$.fragment),oo=i(),xr=s("span"),no=n("Multi-modal processors"),Ns=i(),Gt=s("p"),lo=n(`Any multi-modal model will require an object to encode or decode the data that groups several modalities (among text,
vision and audio). This is handled by objects called processors, which group together two or more processing objects
such as tokenizers (for the text modality), image processors (for vision) and feature extractors (for audio).`),Ms=i(),Rt=s("p"),io=n("Those processors inherit from the following base class that implements the saving and loading functionality:"),zs=i(),L=s("div"),g(Fe.$$.fragment),po=i(),Er=s("p"),co=n("This is a mixin used to provide saving/loading functionality for all processor classes."),mo=i(),X=s("div"),g(Ge.$$.fragment),fo=i(),wr=s("p"),uo=n("Instantiate a processor associated with a pretrained model."),ho=i(),g(ce.$$.fragment),_o=i(),B=s("div"),g(Re.$$.fragment),go=i(),He=s("p"),vo=n(`Upload the processor files to the \u{1F917} Model Hub while synchronizing a local clone of the repo in
`),yr=s("code"),$o=n("repo_path_or_name"),bo=n("."),xo=i(),g(me.$$.fragment),Eo=i(),J=s("div"),g(We.$$.fragment),wo=i(),Xe=s("p"),yo=n(`Register this class with a given auto class. This should only be used for custom feature extractors as the ones
in the library are already mapped with `),Pr=s("code"),Po=n("AutoProcessor"),qo=n("."),ko=i(),g(fe.$$.fragment),Io=i(),K=s("div"),g(Be.$$.fragment),So=i(),Je=s("p"),Ao=n(`Saves the attributes of this processor (feature extractor, tokenizer\u2026) in the specified directory so that it
can be reloaded using the `),Ht=s("a"),Lo=n("from_pretrained()"),Do=n(" method."),To=i(),g(ue.$$.fragment),Vs=i(),re=s("h2"),he=s("a"),qr=s("span"),g(Ke.$$.fragment),No=i(),kr=s("span"),Mo=n("Deprecated processors"),Cs=i(),V=s("p"),zo=n(`All processors follow the same architecture which is that of the
`),Wt=s("a"),Vo=n("DataProcessor"),Co=n(`. The processor returns a list of
`),Xt=s("a"),jo=n("InputExample"),Oo=n(`. These
`),Bt=s("a"),Qo=n("InputExample"),Uo=n(` can be converted to
`),Jt=s("a"),Fo=n("InputFeatures"),Go=n(" in order to be fed to the model."),js=i(),I=s("div"),g(Ye.$$.fragment),Ro=i(),Ir=s("p"),Ho=n("Base class for data converters for sequence classification data sets."),Wo=i(),_e=s("div"),g(Ze.$$.fragment),Xo=i(),et=s("p"),Bo=n("Gets a collection of "),Kt=s("a"),Jo=n("InputExample"),Ko=n(" for the dev set."),Yo=i(),ge=s("div"),g(tt.$$.fragment),Zo=i(),Sr=s("p"),en=n("Gets an example from a dict with tensorflow tensors."),tn=i(),ve=s("div"),g(rt.$$.fragment),rn=i(),Ar=s("p"),sn=n("Gets the list of labels for this data set."),an=i(),$e=s("div"),g(st.$$.fragment),on=i(),at=s("p"),nn=n("Gets a collection of "),Yt=s("a"),ln=n("InputExample"),dn=n(" for the test set."),pn=i(),be=s("div"),g(ot.$$.fragment),cn=i(),nt=s("p"),mn=n("Gets a collection of "),Zt=s("a"),fn=n("InputExample"),un=n(" for the train set."),hn=i(),xe=s("div"),g(lt.$$.fragment),_n=i(),Lr=s("p"),gn=n(`Some tensorflow_datasets datasets are not formatted the same way the GLUE datasets are. This method converts
examples to the correct format.`),Os=i(),G=s("div"),g(it.$$.fragment),vn=i(),Dr=s("p"),$n=n("A single training/test example for simple sequence classification."),bn=i(),Ee=s("div"),g(dt.$$.fragment),xn=i(),Tr=s("p"),En=n("Serializes this instance to a JSON string."),Qs=i(),R=s("div"),g(pt.$$.fragment),wn=i(),Nr=s("p"),yn=n("A single set of features of data. Property names are the same names as the corresponding inputs to a model."),Pn=i(),we=s("div"),g(ct.$$.fragment),qn=i(),Mr=s("p"),kn=n("Serializes this instance to a JSON string."),Us=i(),se=s("h2"),ye=s("a"),zr=s("span"),g(mt.$$.fragment),In=i(),Vr=s("span"),Sn=n("GLUE"),Fs=i(),Pe=s("p"),ft=s("a"),An=n("General Language Understanding Evaluation (GLUE)"),Ln=n(` is a benchmark that evaluates the
performance of models across a diverse set of existing NLU tasks. It was released together with the paper `),ut=s("a"),Dn=n(`GLUE: A
multi-task benchmark and analysis platform for natural language understanding`),Gs=i(),er=s("p"),Tn=n(`This library hosts a total of 10 processors for the following tasks: MRPC, MNLI, MNLI (mismatched), CoLA, SST2, STSB,
QQP, QNLI, RTE and WNLI.`),Rs=i(),tr=s("p"),Nn=n("Those processors are:"),Hs=i(),q=s("ul"),Cr=s("li"),jr=s("code"),Mn=n("~data.processors.utils.MrpcProcessor"),zn=i(),Or=s("li"),Qr=s("code"),Vn=n("~data.processors.utils.MnliProcessor"),Cn=i(),Ur=s("li"),Fr=s("code"),jn=n("~data.processors.utils.MnliMismatchedProcessor"),On=i(),Gr=s("li"),Rr=s("code"),Qn=n("~data.processors.utils.Sst2Processor"),Un=i(),Hr=s("li"),Wr=s("code"),Fn=n("~data.processors.utils.StsbProcessor"),Gn=i(),Xr=s("li"),Br=s("code"),Rn=n("~data.processors.utils.QqpProcessor"),Hn=i(),Jr=s("li"),Kr=s("code"),Wn=n("~data.processors.utils.QnliProcessor"),Xn=i(),Yr=s("li"),Zr=s("code"),Bn=n("~data.processors.utils.RteProcessor"),Jn=i(),es=s("li"),ts=s("code"),Kn=n("~data.processors.utils.WnliProcessor"),Ws=i(),qe=s("p"),Yn=n(`Additionally, the following method can be used to load values from a data file and convert them to a list of
`),rr=s("a"),Zn=n("InputExample"),el=n("."),Xs=i(),ae=s("div"),g(ht.$$.fragment),tl=i(),sr=s("p"),rl=n("Loads a data file into a list of "),rs=s("code"),sl=n("InputFeatures"),Bs=i(),oe=s("h2"),ke=s("a"),ss=s("span"),g(_t.$$.fragment),al=i(),as=s("span"),ol=n("XNLI"),Js=i(),ne=s("p"),gt=s("a"),nl=n("The Cross-Lingual NLI Corpus (XNLI)"),ll=n(` is a benchmark that evaluates the
quality of cross-lingual text representations. XNLI is crowd-sourced dataset based on `),vt=s("a"),os=s("em"),il=n("MultiNLI"),dl=n(`: pairs of text are labeled with textual entailment annotations for 15
different languages (including both high-resource language such as English and low-resource languages such as Swahili).`),Ks=i(),$t=s("p"),pl=n("It was released together with the paper "),bt=s("a"),cl=n("XNLI: Evaluating Cross-lingual Sentence Representations"),Ys=i(),ar=s("p"),ml=n("This library hosts the processor to load the XNLI data:"),Zs=i(),or=s("ul"),ns=s("li"),ls=s("code"),fl=n("~data.processors.utils.XnliProcessor"),ea=i(),nr=s("p"),ul=n("Please note that since the gold labels are available on the test set, evaluation is performed on the test set."),ta=i(),Ie=s("p"),hl=n("An example using these processors is given in the "),xt=s("a"),_l=n("run_xnli.py"),gl=n(" script."),ra=i(),le=s("h2"),Se=s("a"),is=s("span"),g(Et.$$.fragment),vl=i(),ds=s("span"),$l=n("SQuAD"),sa=i(),H=s("p"),wt=s("a"),bl=n("The Stanford Question Answering Dataset (SQuAD)"),xl=n(` is a benchmark that
evaluates the performance of models on question answering. Two versions are available, v1.1 and v2.0. The first version
(v1.1) was released together with the paper `),yt=s("a"),El=n("SQuAD: 100,000+ Questions for Machine Comprehension of Text"),wl=n(". The second version (v2.0) was released alongside the paper "),Pt=s("a"),yl=n(`Know What You Don\u2019t
Know: Unanswerable Questions for SQuAD`),Pl=n("."),aa=i(),lr=s("p"),ql=n("This library hosts a processor for each of the two versions:"),oa=i(),ie=s("h3"),Ae=s("a"),ps=s("span"),g(qt.$$.fragment),kl=i(),cs=s("span"),Il=n("Processors"),na=i(),ir=s("p"),Sl=n("Those processors are:"),la=i(),Le=s("ul"),ms=s("li"),fs=s("code"),Al=n("~data.processors.utils.SquadV1Processor"),Ll=i(),us=s("li"),hs=s("code"),Dl=n("~data.processors.utils.SquadV2Processor"),ia=i(),kt=s("p"),Tl=n("They both inherit from the abstract class "),_s=s("code"),Nl=n("~data.processors.utils.SquadProcessor"),da=i(),M=s("div"),g(It.$$.fragment),Ml=i(),gs=s("p"),zl=n(`Processor for the SQuAD data set. overridden by SquadV1Processor and SquadV2Processor, used by the version 1.1 and
version 2.0 of SQuAD, respectively.`),Vl=i(),De=s("div"),g(St.$$.fragment),Cl=i(),vs=s("p"),jl=n("Returns the evaluation example from the data directory."),Ol=i(),Y=s("div"),g(At.$$.fragment),Ql=i(),Lt=s("p"),Ul=n("Creates a list of "),$s=s("code"),Fl=n("SquadExample"),Gl=n(" using a TFDS dataset."),Rl=i(),g(Te.$$.fragment),Hl=i(),Ne=s("div"),g(Dt.$$.fragment),Wl=i(),bs=s("p"),Xl=n("Returns the training examples from the data directory."),pa=i(),Me=s("p"),Bl=n(`Additionally, the following method can be used to convert SQuAD examples into
`),xs=s("code"),Jl=n("~data.processors.utils.SquadFeatures"),Kl=n(" that can be used as model inputs."),ca=i(),W=s("div"),g(Tt.$$.fragment),Yl=i(),Es=s("p"),Zl=n(`Converts a list of examples into a list of features that can be directly given as input to a model. It is
model-dependant and takes advantage of many of the tokenizer\u2019s features to create the model\u2019s inputs.`),ei=i(),g(ze.$$.fragment),ma=i(),Ve=s("p"),ti=n(`These processors as well as the aforementioned method can be used with files containing the data as well as with the
`),ws=s("em"),ri=n("tensorflow_datasets"),si=n(" package. Examples are given below."),fa=i(),de=s("h3"),Ce=s("a"),ys=s("span"),g(Nt.$$.fragment),ai=i(),Ps=s("span"),oi=n("Example usage"),ua=i(),dr=s("p"),ni=n("Here is an example using the processors as well as the conversion method using data files:"),ha=i(),g(Mt.$$.fragment),_a=i(),je=s("p"),li=n("Using "),qs=s("em"),ii=n("tensorflow_datasets"),di=n(" is as easy as using a data file:"),ga=i(),g(zt.$$.fragment),va=i(),Oe=s("p"),pi=n("Another example using these processors is given in the "),Vt=s("a"),ci=n("run_squad.py"),mi=n(" script."),this.h()},l(e){const p=up('[data-svelte="svelte-1phssyn"]',document.head);u=a(p,"META",{name:!0,content:!0}),p.forEach(r),y=d(e),_=a(e,"H1",{class:!0});var Ct=o(_);h=a(Ct,"A",{id:!0,class:!0,href:!0});var ks=o(h);P=a(ks,"SPAN",{});var Is=o(P);v(f.$$.fragment,Is),Is.forEach(r),ks.forEach(r),w=d(Ct),C=a(Ct,"SPAN",{});var Ss=o(C);T=l(Ss,"Processors"),Ss.forEach(r),Ct.forEach(r),A=d(e),j=a(e,"P",{});var As=o(j);O=l(As,"Processors can mean two different things in the Transformers library:"),As.forEach(r),N=d(e),F=a(e,"UL",{});var jt=o(F);Q=a(jt,"LI",{});var pr=o(Q);Qt=l(pr,"the objects that pre-process inputs for multi-modal models such as "),Ut=a(pr,"A",{href:!0});var $i=o(Ut);Za=l($i,"Wav2Vec2"),$i.forEach(r),eo=l(pr,` (speech and text)
or `),Ft=a(pr,"A",{href:!0});var bi=o(Ft);to=l(bi,"CLIP"),bi.forEach(r),ro=l(pr," (text and vision)"),pr.forEach(r),so=d(jt),$r=a(jt,"LI",{});var xi=o($r);ao=l(xi,"deprecated objects that were used in older versions of the library to preprocess data for GLUE or SQUAD."),xi.forEach(r),jt.forEach(r),Ts=d(e),te=a(e,"H2",{class:!0});var ba=o(te);pe=a(ba,"A",{id:!0,class:!0,href:!0});var Ei=o(pe);br=a(Ei,"SPAN",{});var wi=o(br);v(Ue.$$.fragment,wi),wi.forEach(r),Ei.forEach(r),oo=d(ba),xr=a(ba,"SPAN",{});var yi=o(xr);no=l(yi,"Multi-modal processors"),yi.forEach(r),ba.forEach(r),Ns=d(e),Gt=a(e,"P",{});var Pi=o(Gt);lo=l(Pi,`Any multi-modal model will require an object to encode or decode the data that groups several modalities (among text,
vision and audio). This is handled by objects called processors, which group together two or more processing objects
such as tokenizers (for the text modality), image processors (for vision) and feature extractors (for audio).`),Pi.forEach(r),Ms=d(e),Rt=a(e,"P",{});var qi=o(Rt);io=l(qi,"Those processors inherit from the following base class that implements the saving and loading functionality:"),qi.forEach(r),zs=d(e),L=a(e,"DIV",{class:!0});var U=o(L);v(Fe.$$.fragment,U),po=d(U),Er=a(U,"P",{});var ki=o(Er);co=l(ki,"This is a mixin used to provide saving/loading functionality for all processor classes."),ki.forEach(r),mo=d(U),X=a(U,"DIV",{class:!0});var cr=o(X);v(Ge.$$.fragment,cr),fo=d(cr),wr=a(cr,"P",{});var Ii=o(wr);uo=l(Ii,"Instantiate a processor associated with a pretrained model."),Ii.forEach(r),ho=d(cr),v(ce.$$.fragment,cr),cr.forEach(r),_o=d(U),B=a(U,"DIV",{class:!0});var mr=o(B);v(Re.$$.fragment,mr),go=d(mr),He=a(mr,"P",{});var xa=o(He);vo=l(xa,`Upload the processor files to the \u{1F917} Model Hub while synchronizing a local clone of the repo in
`),yr=a(xa,"CODE",{});var Si=o(yr);$o=l(Si,"repo_path_or_name"),Si.forEach(r),bo=l(xa,"."),xa.forEach(r),xo=d(mr),v(me.$$.fragment,mr),mr.forEach(r),Eo=d(U),J=a(U,"DIV",{class:!0});var fr=o(J);v(We.$$.fragment,fr),wo=d(fr),Xe=a(fr,"P",{});var Ea=o(Xe);yo=l(Ea,`Register this class with a given auto class. This should only be used for custom feature extractors as the ones
in the library are already mapped with `),Pr=a(Ea,"CODE",{});var Ai=o(Pr);Po=l(Ai,"AutoProcessor"),Ai.forEach(r),qo=l(Ea,"."),Ea.forEach(r),ko=d(fr),v(fe.$$.fragment,fr),fr.forEach(r),Io=d(U),K=a(U,"DIV",{class:!0});var ur=o(K);v(Be.$$.fragment,ur),So=d(ur),Je=a(ur,"P",{});var wa=o(Je);Ao=l(wa,`Saves the attributes of this processor (feature extractor, tokenizer\u2026) in the specified directory so that it
can be reloaded using the `),Ht=a(wa,"A",{href:!0});var Li=o(Ht);Lo=l(Li,"from_pretrained()"),Li.forEach(r),Do=l(wa," method."),wa.forEach(r),To=d(ur),v(ue.$$.fragment,ur),ur.forEach(r),U.forEach(r),Vs=d(e),re=a(e,"H2",{class:!0});var ya=o(re);he=a(ya,"A",{id:!0,class:!0,href:!0});var Di=o(he);qr=a(Di,"SPAN",{});var Ti=o(qr);v(Ke.$$.fragment,Ti),Ti.forEach(r),Di.forEach(r),No=d(ya),kr=a(ya,"SPAN",{});var Ni=o(kr);Mo=l(Ni,"Deprecated processors"),Ni.forEach(r),ya.forEach(r),Cs=d(e),V=a(e,"P",{});var Z=o(V);zo=l(Z,`All processors follow the same architecture which is that of the
`),Wt=a(Z,"A",{href:!0});var Mi=o(Wt);Vo=l(Mi,"DataProcessor"),Mi.forEach(r),Co=l(Z,`. The processor returns a list of
`),Xt=a(Z,"A",{href:!0});var zi=o(Xt);jo=l(zi,"InputExample"),zi.forEach(r),Oo=l(Z,`. These
`),Bt=a(Z,"A",{href:!0});var Vi=o(Bt);Qo=l(Vi,"InputExample"),Vi.forEach(r),Uo=l(Z,` can be converted to
`),Jt=a(Z,"A",{href:!0});var Ci=o(Jt);Fo=l(Ci,"InputFeatures"),Ci.forEach(r),Go=l(Z," in order to be fed to the model."),Z.forEach(r),js=d(e),I=a(e,"DIV",{class:!0});var D=o(I);v(Ye.$$.fragment,D),Ro=d(D),Ir=a(D,"P",{});var ji=o(Ir);Ho=l(ji,"Base class for data converters for sequence classification data sets."),ji.forEach(r),Wo=d(D),_e=a(D,"DIV",{class:!0});var Pa=o(_e);v(Ze.$$.fragment,Pa),Xo=d(Pa),et=a(Pa,"P",{});var qa=o(et);Bo=l(qa,"Gets a collection of "),Kt=a(qa,"A",{href:!0});var Oi=o(Kt);Jo=l(Oi,"InputExample"),Oi.forEach(r),Ko=l(qa," for the dev set."),qa.forEach(r),Pa.forEach(r),Yo=d(D),ge=a(D,"DIV",{class:!0});var ka=o(ge);v(tt.$$.fragment,ka),Zo=d(ka),Sr=a(ka,"P",{});var Qi=o(Sr);en=l(Qi,"Gets an example from a dict with tensorflow tensors."),Qi.forEach(r),ka.forEach(r),tn=d(D),ve=a(D,"DIV",{class:!0});var Ia=o(ve);v(rt.$$.fragment,Ia),rn=d(Ia),Ar=a(Ia,"P",{});var Ui=o(Ar);sn=l(Ui,"Gets the list of labels for this data set."),Ui.forEach(r),Ia.forEach(r),an=d(D),$e=a(D,"DIV",{class:!0});var Sa=o($e);v(st.$$.fragment,Sa),on=d(Sa),at=a(Sa,"P",{});var Aa=o(at);nn=l(Aa,"Gets a collection of "),Yt=a(Aa,"A",{href:!0});var Fi=o(Yt);ln=l(Fi,"InputExample"),Fi.forEach(r),dn=l(Aa," for the test set."),Aa.forEach(r),Sa.forEach(r),pn=d(D),be=a(D,"DIV",{class:!0});var La=o(be);v(ot.$$.fragment,La),cn=d(La),nt=a(La,"P",{});var Da=o(nt);mn=l(Da,"Gets a collection of "),Zt=a(Da,"A",{href:!0});var Gi=o(Zt);fn=l(Gi,"InputExample"),Gi.forEach(r),un=l(Da," for the train set."),Da.forEach(r),La.forEach(r),hn=d(D),xe=a(D,"DIV",{class:!0});var Ta=o(xe);v(lt.$$.fragment,Ta),_n=d(Ta),Lr=a(Ta,"P",{});var Ri=o(Lr);gn=l(Ri,`Some tensorflow_datasets datasets are not formatted the same way the GLUE datasets are. This method converts
examples to the correct format.`),Ri.forEach(r),Ta.forEach(r),D.forEach(r),Os=d(e),G=a(e,"DIV",{class:!0});var hr=o(G);v(it.$$.fragment,hr),vn=d(hr),Dr=a(hr,"P",{});var Hi=o(Dr);$n=l(Hi,"A single training/test example for simple sequence classification."),Hi.forEach(r),bn=d(hr),Ee=a(hr,"DIV",{class:!0});var Na=o(Ee);v(dt.$$.fragment,Na),xn=d(Na),Tr=a(Na,"P",{});var Wi=o(Tr);En=l(Wi,"Serializes this instance to a JSON string."),Wi.forEach(r),Na.forEach(r),hr.forEach(r),Qs=d(e),R=a(e,"DIV",{class:!0});var _r=o(R);v(pt.$$.fragment,_r),wn=d(_r),Nr=a(_r,"P",{});var Xi=o(Nr);yn=l(Xi,"A single set of features of data. Property names are the same names as the corresponding inputs to a model."),Xi.forEach(r),Pn=d(_r),we=a(_r,"DIV",{class:!0});var Ma=o(we);v(ct.$$.fragment,Ma),qn=d(Ma),Mr=a(Ma,"P",{});var Bi=o(Mr);kn=l(Bi,"Serializes this instance to a JSON string."),Bi.forEach(r),Ma.forEach(r),_r.forEach(r),Us=d(e),se=a(e,"H2",{class:!0});var za=o(se);ye=a(za,"A",{id:!0,class:!0,href:!0});var Ji=o(ye);zr=a(Ji,"SPAN",{});var Ki=o(zr);v(mt.$$.fragment,Ki),Ki.forEach(r),Ji.forEach(r),In=d(za),Vr=a(za,"SPAN",{});var Yi=o(Vr);Sn=l(Yi,"GLUE"),Yi.forEach(r),za.forEach(r),Fs=d(e),Pe=a(e,"P",{});var Va=o(Pe);ft=a(Va,"A",{href:!0,rel:!0});var Zi=o(ft);An=l(Zi,"General Language Understanding Evaluation (GLUE)"),Zi.forEach(r),Ln=l(Va,` is a benchmark that evaluates the
performance of models across a diverse set of existing NLU tasks. It was released together with the paper `),ut=a(Va,"A",{href:!0,rel:!0});var ed=o(ut);Dn=l(ed,`GLUE: A
multi-task benchmark and analysis platform for natural language understanding`),ed.forEach(r),Va.forEach(r),Gs=d(e),er=a(e,"P",{});var td=o(er);Tn=l(td,`This library hosts a total of 10 processors for the following tasks: MRPC, MNLI, MNLI (mismatched), CoLA, SST2, STSB,
QQP, QNLI, RTE and WNLI.`),td.forEach(r),Rs=d(e),tr=a(e,"P",{});var rd=o(tr);Nn=l(rd,"Those processors are:"),rd.forEach(r),Hs=d(e),q=a(e,"UL",{});var S=o(q);Cr=a(S,"LI",{});var sd=o(Cr);jr=a(sd,"CODE",{});var ad=o(jr);Mn=l(ad,"~data.processors.utils.MrpcProcessor"),ad.forEach(r),sd.forEach(r),zn=d(S),Or=a(S,"LI",{});var od=o(Or);Qr=a(od,"CODE",{});var nd=o(Qr);Vn=l(nd,"~data.processors.utils.MnliProcessor"),nd.forEach(r),od.forEach(r),Cn=d(S),Ur=a(S,"LI",{});var ld=o(Ur);Fr=a(ld,"CODE",{});var id=o(Fr);jn=l(id,"~data.processors.utils.MnliMismatchedProcessor"),id.forEach(r),ld.forEach(r),On=d(S),Gr=a(S,"LI",{});var dd=o(Gr);Rr=a(dd,"CODE",{});var pd=o(Rr);Qn=l(pd,"~data.processors.utils.Sst2Processor"),pd.forEach(r),dd.forEach(r),Un=d(S),Hr=a(S,"LI",{});var cd=o(Hr);Wr=a(cd,"CODE",{});var md=o(Wr);Fn=l(md,"~data.processors.utils.StsbProcessor"),md.forEach(r),cd.forEach(r),Gn=d(S),Xr=a(S,"LI",{});var fd=o(Xr);Br=a(fd,"CODE",{});var ud=o(Br);Rn=l(ud,"~data.processors.utils.QqpProcessor"),ud.forEach(r),fd.forEach(r),Hn=d(S),Jr=a(S,"LI",{});var hd=o(Jr);Kr=a(hd,"CODE",{});var _d=o(Kr);Wn=l(_d,"~data.processors.utils.QnliProcessor"),_d.forEach(r),hd.forEach(r),Xn=d(S),Yr=a(S,"LI",{});var gd=o(Yr);Zr=a(gd,"CODE",{});var vd=o(Zr);Bn=l(vd,"~data.processors.utils.RteProcessor"),vd.forEach(r),gd.forEach(r),Jn=d(S),es=a(S,"LI",{});var $d=o(es);ts=a($d,"CODE",{});var bd=o(ts);Kn=l(bd,"~data.processors.utils.WnliProcessor"),bd.forEach(r),$d.forEach(r),S.forEach(r),Ws=d(e),qe=a(e,"P",{});var Ca=o(qe);Yn=l(Ca,`Additionally, the following method can be used to load values from a data file and convert them to a list of
`),rr=a(Ca,"A",{href:!0});var xd=o(rr);Zn=l(xd,"InputExample"),xd.forEach(r),el=l(Ca,"."),Ca.forEach(r),Xs=d(e),ae=a(e,"DIV",{class:!0});var ja=o(ae);v(ht.$$.fragment,ja),tl=d(ja),sr=a(ja,"P",{});var fi=o(sr);rl=l(fi,"Loads a data file into a list of "),rs=a(fi,"CODE",{});var Ed=o(rs);sl=l(Ed,"InputFeatures"),Ed.forEach(r),fi.forEach(r),ja.forEach(r),Bs=d(e),oe=a(e,"H2",{class:!0});var Oa=o(oe);ke=a(Oa,"A",{id:!0,class:!0,href:!0});var wd=o(ke);ss=a(wd,"SPAN",{});var yd=o(ss);v(_t.$$.fragment,yd),yd.forEach(r),wd.forEach(r),al=d(Oa),as=a(Oa,"SPAN",{});var Pd=o(as);ol=l(Pd,"XNLI"),Pd.forEach(r),Oa.forEach(r),Js=d(e),ne=a(e,"P",{});var Ls=o(ne);gt=a(Ls,"A",{href:!0,rel:!0});var qd=o(gt);nl=l(qd,"The Cross-Lingual NLI Corpus (XNLI)"),qd.forEach(r),ll=l(Ls,` is a benchmark that evaluates the
quality of cross-lingual text representations. XNLI is crowd-sourced dataset based on `),vt=a(Ls,"A",{href:!0,rel:!0});var kd=o(vt);os=a(kd,"EM",{});var Id=o(os);il=l(Id,"MultiNLI"),Id.forEach(r),kd.forEach(r),dl=l(Ls,`: pairs of text are labeled with textual entailment annotations for 15
different languages (including both high-resource language such as English and low-resource languages such as Swahili).`),Ls.forEach(r),Ks=d(e),$t=a(e,"P",{});var ui=o($t);pl=l(ui,"It was released together with the paper "),bt=a(ui,"A",{href:!0,rel:!0});var Sd=o(bt);cl=l(Sd,"XNLI: Evaluating Cross-lingual Sentence Representations"),Sd.forEach(r),ui.forEach(r),Ys=d(e),ar=a(e,"P",{});var Ad=o(ar);ml=l(Ad,"This library hosts the processor to load the XNLI data:"),Ad.forEach(r),Zs=d(e),or=a(e,"UL",{});var Ld=o(or);ns=a(Ld,"LI",{});var Dd=o(ns);ls=a(Dd,"CODE",{});var Td=o(ls);fl=l(Td,"~data.processors.utils.XnliProcessor"),Td.forEach(r),Dd.forEach(r),Ld.forEach(r),ea=d(e),nr=a(e,"P",{});var Nd=o(nr);ul=l(Nd,"Please note that since the gold labels are available on the test set, evaluation is performed on the test set."),Nd.forEach(r),ta=d(e),Ie=a(e,"P",{});var Qa=o(Ie);hl=l(Qa,"An example using these processors is given in the "),xt=a(Qa,"A",{href:!0,rel:!0});var Md=o(xt);_l=l(Md,"run_xnli.py"),Md.forEach(r),gl=l(Qa," script."),Qa.forEach(r),ra=d(e),le=a(e,"H2",{class:!0});var Ua=o(le);Se=a(Ua,"A",{id:!0,class:!0,href:!0});var zd=o(Se);is=a(zd,"SPAN",{});var Vd=o(is);v(Et.$$.fragment,Vd),Vd.forEach(r),zd.forEach(r),vl=d(Ua),ds=a(Ua,"SPAN",{});var Cd=o(ds);$l=l(Cd,"SQuAD"),Cd.forEach(r),Ua.forEach(r),sa=d(e),H=a(e,"P",{});var Ot=o(H);wt=a(Ot,"A",{href:!0,rel:!0});var jd=o(wt);bl=l(jd,"The Stanford Question Answering Dataset (SQuAD)"),jd.forEach(r),xl=l(Ot,` is a benchmark that
evaluates the performance of models on question answering. Two versions are available, v1.1 and v2.0. The first version
(v1.1) was released together with the paper `),yt=a(Ot,"A",{href:!0,rel:!0});var Od=o(yt);El=l(Od,"SQuAD: 100,000+ Questions for Machine Comprehension of Text"),Od.forEach(r),wl=l(Ot,". The second version (v2.0) was released alongside the paper "),Pt=a(Ot,"A",{href:!0,rel:!0});var Qd=o(Pt);yl=l(Qd,`Know What You Don\u2019t
Know: Unanswerable Questions for SQuAD`),Qd.forEach(r),Pl=l(Ot,"."),Ot.forEach(r),aa=d(e),lr=a(e,"P",{});var Ud=o(lr);ql=l(Ud,"This library hosts a processor for each of the two versions:"),Ud.forEach(r),oa=d(e),ie=a(e,"H3",{class:!0});var Fa=o(ie);Ae=a(Fa,"A",{id:!0,class:!0,href:!0});var Fd=o(Ae);ps=a(Fd,"SPAN",{});var Gd=o(ps);v(qt.$$.fragment,Gd),Gd.forEach(r),Fd.forEach(r),kl=d(Fa),cs=a(Fa,"SPAN",{});var Rd=o(cs);Il=l(Rd,"Processors"),Rd.forEach(r),Fa.forEach(r),na=d(e),ir=a(e,"P",{});var Hd=o(ir);Sl=l(Hd,"Those processors are:"),Hd.forEach(r),la=d(e),Le=a(e,"UL",{});var Ga=o(Le);ms=a(Ga,"LI",{});var Wd=o(ms);fs=a(Wd,"CODE",{});var Xd=o(fs);Al=l(Xd,"~data.processors.utils.SquadV1Processor"),Xd.forEach(r),Wd.forEach(r),Ll=d(Ga),us=a(Ga,"LI",{});var Bd=o(us);hs=a(Bd,"CODE",{});var Jd=o(hs);Dl=l(Jd,"~data.processors.utils.SquadV2Processor"),Jd.forEach(r),Bd.forEach(r),Ga.forEach(r),ia=d(e),kt=a(e,"P",{});var hi=o(kt);Tl=l(hi,"They both inherit from the abstract class "),_s=a(hi,"CODE",{});var Kd=o(_s);Nl=l(Kd,"~data.processors.utils.SquadProcessor"),Kd.forEach(r),hi.forEach(r),da=d(e),M=a(e,"DIV",{class:!0});var ee=o(M);v(It.$$.fragment,ee),Ml=d(ee),gs=a(ee,"P",{});var Yd=o(gs);zl=l(Yd,`Processor for the SQuAD data set. overridden by SquadV1Processor and SquadV2Processor, used by the version 1.1 and
version 2.0 of SQuAD, respectively.`),Yd.forEach(r),Vl=d(ee),De=a(ee,"DIV",{class:!0});var Ra=o(De);v(St.$$.fragment,Ra),Cl=d(Ra),vs=a(Ra,"P",{});var Zd=o(vs);jl=l(Zd,"Returns the evaluation example from the data directory."),Zd.forEach(r),Ra.forEach(r),Ol=d(ee),Y=a(ee,"DIV",{class:!0});var gr=o(Y);v(At.$$.fragment,gr),Ql=d(gr),Lt=a(gr,"P",{});var Ha=o(Lt);Ul=l(Ha,"Creates a list of "),$s=a(Ha,"CODE",{});var ep=o($s);Fl=l(ep,"SquadExample"),ep.forEach(r),Gl=l(Ha," using a TFDS dataset."),Ha.forEach(r),Rl=d(gr),v(Te.$$.fragment,gr),gr.forEach(r),Hl=d(ee),Ne=a(ee,"DIV",{class:!0});var Wa=o(Ne);v(Dt.$$.fragment,Wa),Wl=d(Wa),bs=a(Wa,"P",{});var tp=o(bs);Xl=l(tp,"Returns the training examples from the data directory."),tp.forEach(r),Wa.forEach(r),ee.forEach(r),pa=d(e),Me=a(e,"P",{});var Xa=o(Me);Bl=l(Xa,`Additionally, the following method can be used to convert SQuAD examples into
`),xs=a(Xa,"CODE",{});var rp=o(xs);Jl=l(rp,"~data.processors.utils.SquadFeatures"),rp.forEach(r),Kl=l(Xa," that can be used as model inputs."),Xa.forEach(r),ca=d(e),W=a(e,"DIV",{class:!0});var vr=o(W);v(Tt.$$.fragment,vr),Yl=d(vr),Es=a(vr,"P",{});var sp=o(Es);Zl=l(sp,`Converts a list of examples into a list of features that can be directly given as input to a model. It is
model-dependant and takes advantage of many of the tokenizer\u2019s features to create the model\u2019s inputs.`),sp.forEach(r),ei=d(vr),v(ze.$$.fragment,vr),vr.forEach(r),ma=d(e),Ve=a(e,"P",{});var Ba=o(Ve);ti=l(Ba,`These processors as well as the aforementioned method can be used with files containing the data as well as with the
`),ws=a(Ba,"EM",{});var ap=o(ws);ri=l(ap,"tensorflow_datasets"),ap.forEach(r),si=l(Ba," package. Examples are given below."),Ba.forEach(r),fa=d(e),de=a(e,"H3",{class:!0});var Ja=o(de);Ce=a(Ja,"A",{id:!0,class:!0,href:!0});var op=o(Ce);ys=a(op,"SPAN",{});var np=o(ys);v(Nt.$$.fragment,np),np.forEach(r),op.forEach(r),ai=d(Ja),Ps=a(Ja,"SPAN",{});var lp=o(Ps);oi=l(lp,"Example usage"),lp.forEach(r),Ja.forEach(r),ua=d(e),dr=a(e,"P",{});var ip=o(dr);ni=l(ip,"Here is an example using the processors as well as the conversion method using data files:"),ip.forEach(r),ha=d(e),v(Mt.$$.fragment,e),_a=d(e),je=a(e,"P",{});var Ka=o(je);li=l(Ka,"Using "),qs=a(Ka,"EM",{});var dp=o(qs);ii=l(dp,"tensorflow_datasets"),dp.forEach(r),di=l(Ka," is as easy as using a data file:"),Ka.forEach(r),ga=d(e),v(zt.$$.fragment,e),va=d(e),Oe=a(e,"P",{});var Ya=o(Oe);pi=l(Ya,"Another example using these processors is given in the "),Vt=a(Ya,"A",{href:!0,rel:!0});var pp=o(Vt);ci=l(pp,"run_squad.py"),pp.forEach(r),mi=l(Ya," script."),Ya.forEach(r),this.h()},h(){m(u,"name","hf:doc:metadata"),m(u,"content",JSON.stringify(wp)),m(h,"id","processors"),m(h,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(h,"href","#processors"),m(_,"class","relative group"),m(Ut,"href","../model_doc/wav2vec2"),m(Ft,"href","../model_doc/clip"),m(pe,"id","transformers.ProcessorMixin"),m(pe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(pe,"href","#transformers.ProcessorMixin"),m(te,"class","relative group"),m(X,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(B,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(J,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(Ht,"href","/docs/transformers/main/en/model_doc/trocr#transformers.TrOCRProcessor.from_pretrained"),m(K,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(L,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(he,"id","transformers.DataProcessor"),m(he,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(he,"href","#transformers.DataProcessor"),m(re,"class","relative group"),m(Wt,"href","/docs/transformers/main/en/main_classes/processors#transformers.DataProcessor"),m(Xt,"href","/docs/transformers/main/en/main_classes/processors#transformers.InputExample"),m(Bt,"href","/docs/transformers/main/en/main_classes/processors#transformers.InputExample"),m(Jt,"href","/docs/transformers/main/en/main_classes/processors#transformers.InputFeatures"),m(Kt,"href","/docs/transformers/main/en/main_classes/processors#transformers.InputExample"),m(_e,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(ge,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(ve,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(Yt,"href","/docs/transformers/main/en/main_classes/processors#transformers.InputExample"),m($e,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(Zt,"href","/docs/transformers/main/en/main_classes/processors#transformers.InputExample"),m(be,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(xe,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(I,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(Ee,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(G,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(we,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(R,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(ye,"id","transformers.glue_convert_examples_to_features"),m(ye,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(ye,"href","#transformers.glue_convert_examples_to_features"),m(se,"class","relative group"),m(ft,"href","https://gluebenchmark.com/"),m(ft,"rel","nofollow"),m(ut,"href","https://openreview.net/pdf?id=rJ4km2R5t7"),m(ut,"rel","nofollow"),m(rr,"href","/docs/transformers/main/en/main_classes/processors#transformers.InputExample"),m(ae,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(ke,"id","xnli"),m(ke,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(ke,"href","#xnli"),m(oe,"class","relative group"),m(gt,"href","https://www.nyu.edu/projects/bowman/xnli/"),m(gt,"rel","nofollow"),m(vt,"href","http://www.nyu.edu/projects/bowman/multinli/"),m(vt,"rel","nofollow"),m(bt,"href","https://arxiv.org/abs/1809.05053"),m(bt,"rel","nofollow"),m(xt,"href","https://github.com/huggingface/transformers/tree/main/examples/legacy/text-classification/run_xnli.py"),m(xt,"rel","nofollow"),m(Se,"id","squad"),m(Se,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Se,"href","#squad"),m(le,"class","relative group"),m(wt,"href","https://rajpurkar.github.io/SQuAD-explorer//"),m(wt,"rel","nofollow"),m(yt,"href","https://arxiv.org/abs/1606.05250"),m(yt,"rel","nofollow"),m(Pt,"href","https://arxiv.org/abs/1806.03822"),m(Pt,"rel","nofollow"),m(Ae,"id","transformers.data.processors.squad.SquadProcessor"),m(Ae,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Ae,"href","#transformers.data.processors.squad.SquadProcessor"),m(ie,"class","relative group"),m(De,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(Y,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(Ne,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(M,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(W,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(Ce,"id","example-usage"),m(Ce,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Ce,"href","#example-usage"),m(de,"class","relative group"),m(Vt,"href","https://github.com/huggingface/transformers/tree/main/examples/legacy/question-answering/run_squad.py"),m(Vt,"rel","nofollow")},m(e,p){t(document.head,u),c(e,y,p),c(e,_,p),t(_,h),t(h,P),$(f,P,null),t(_,w),t(_,C),t(C,T),c(e,A,p),c(e,j,p),t(j,O),c(e,N,p),c(e,F,p),t(F,Q),t(Q,Qt),t(Q,Ut),t(Ut,Za),t(Q,eo),t(Q,Ft),t(Ft,to),t(Q,ro),t(F,so),t(F,$r),t($r,ao),c(e,Ts,p),c(e,te,p),t(te,pe),t(pe,br),$(Ue,br,null),t(te,oo),t(te,xr),t(xr,no),c(e,Ns,p),c(e,Gt,p),t(Gt,lo),c(e,Ms,p),c(e,Rt,p),t(Rt,io),c(e,zs,p),c(e,L,p),$(Fe,L,null),t(L,po),t(L,Er),t(Er,co),t(L,mo),t(L,X),$(Ge,X,null),t(X,fo),t(X,wr),t(wr,uo),t(X,ho),$(ce,X,null),t(L,_o),t(L,B),$(Re,B,null),t(B,go),t(B,He),t(He,vo),t(He,yr),t(yr,$o),t(He,bo),t(B,xo),$(me,B,null),t(L,Eo),t(L,J),$(We,J,null),t(J,wo),t(J,Xe),t(Xe,yo),t(Xe,Pr),t(Pr,Po),t(Xe,qo),t(J,ko),$(fe,J,null),t(L,Io),t(L,K),$(Be,K,null),t(K,So),t(K,Je),t(Je,Ao),t(Je,Ht),t(Ht,Lo),t(Je,Do),t(K,To),$(ue,K,null),c(e,Vs,p),c(e,re,p),t(re,he),t(he,qr),$(Ke,qr,null),t(re,No),t(re,kr),t(kr,Mo),c(e,Cs,p),c(e,V,p),t(V,zo),t(V,Wt),t(Wt,Vo),t(V,Co),t(V,Xt),t(Xt,jo),t(V,Oo),t(V,Bt),t(Bt,Qo),t(V,Uo),t(V,Jt),t(Jt,Fo),t(V,Go),c(e,js,p),c(e,I,p),$(Ye,I,null),t(I,Ro),t(I,Ir),t(Ir,Ho),t(I,Wo),t(I,_e),$(Ze,_e,null),t(_e,Xo),t(_e,et),t(et,Bo),t(et,Kt),t(Kt,Jo),t(et,Ko),t(I,Yo),t(I,ge),$(tt,ge,null),t(ge,Zo),t(ge,Sr),t(Sr,en),t(I,tn),t(I,ve),$(rt,ve,null),t(ve,rn),t(ve,Ar),t(Ar,sn),t(I,an),t(I,$e),$(st,$e,null),t($e,on),t($e,at),t(at,nn),t(at,Yt),t(Yt,ln),t(at,dn),t(I,pn),t(I,be),$(ot,be,null),t(be,cn),t(be,nt),t(nt,mn),t(nt,Zt),t(Zt,fn),t(nt,un),t(I,hn),t(I,xe),$(lt,xe,null),t(xe,_n),t(xe,Lr),t(Lr,gn),c(e,Os,p),c(e,G,p),$(it,G,null),t(G,vn),t(G,Dr),t(Dr,$n),t(G,bn),t(G,Ee),$(dt,Ee,null),t(Ee,xn),t(Ee,Tr),t(Tr,En),c(e,Qs,p),c(e,R,p),$(pt,R,null),t(R,wn),t(R,Nr),t(Nr,yn),t(R,Pn),t(R,we),$(ct,we,null),t(we,qn),t(we,Mr),t(Mr,kn),c(e,Us,p),c(e,se,p),t(se,ye),t(ye,zr),$(mt,zr,null),t(se,In),t(se,Vr),t(Vr,Sn),c(e,Fs,p),c(e,Pe,p),t(Pe,ft),t(ft,An),t(Pe,Ln),t(Pe,ut),t(ut,Dn),c(e,Gs,p),c(e,er,p),t(er,Tn),c(e,Rs,p),c(e,tr,p),t(tr,Nn),c(e,Hs,p),c(e,q,p),t(q,Cr),t(Cr,jr),t(jr,Mn),t(q,zn),t(q,Or),t(Or,Qr),t(Qr,Vn),t(q,Cn),t(q,Ur),t(Ur,Fr),t(Fr,jn),t(q,On),t(q,Gr),t(Gr,Rr),t(Rr,Qn),t(q,Un),t(q,Hr),t(Hr,Wr),t(Wr,Fn),t(q,Gn),t(q,Xr),t(Xr,Br),t(Br,Rn),t(q,Hn),t(q,Jr),t(Jr,Kr),t(Kr,Wn),t(q,Xn),t(q,Yr),t(Yr,Zr),t(Zr,Bn),t(q,Jn),t(q,es),t(es,ts),t(ts,Kn),c(e,Ws,p),c(e,qe,p),t(qe,Yn),t(qe,rr),t(rr,Zn),t(qe,el),c(e,Xs,p),c(e,ae,p),$(ht,ae,null),t(ae,tl),t(ae,sr),t(sr,rl),t(sr,rs),t(rs,sl),c(e,Bs,p),c(e,oe,p),t(oe,ke),t(ke,ss),$(_t,ss,null),t(oe,al),t(oe,as),t(as,ol),c(e,Js,p),c(e,ne,p),t(ne,gt),t(gt,nl),t(ne,ll),t(ne,vt),t(vt,os),t(os,il),t(ne,dl),c(e,Ks,p),c(e,$t,p),t($t,pl),t($t,bt),t(bt,cl),c(e,Ys,p),c(e,ar,p),t(ar,ml),c(e,Zs,p),c(e,or,p),t(or,ns),t(ns,ls),t(ls,fl),c(e,ea,p),c(e,nr,p),t(nr,ul),c(e,ta,p),c(e,Ie,p),t(Ie,hl),t(Ie,xt),t(xt,_l),t(Ie,gl),c(e,ra,p),c(e,le,p),t(le,Se),t(Se,is),$(Et,is,null),t(le,vl),t(le,ds),t(ds,$l),c(e,sa,p),c(e,H,p),t(H,wt),t(wt,bl),t(H,xl),t(H,yt),t(yt,El),t(H,wl),t(H,Pt),t(Pt,yl),t(H,Pl),c(e,aa,p),c(e,lr,p),t(lr,ql),c(e,oa,p),c(e,ie,p),t(ie,Ae),t(Ae,ps),$(qt,ps,null),t(ie,kl),t(ie,cs),t(cs,Il),c(e,na,p),c(e,ir,p),t(ir,Sl),c(e,la,p),c(e,Le,p),t(Le,ms),t(ms,fs),t(fs,Al),t(Le,Ll),t(Le,us),t(us,hs),t(hs,Dl),c(e,ia,p),c(e,kt,p),t(kt,Tl),t(kt,_s),t(_s,Nl),c(e,da,p),c(e,M,p),$(It,M,null),t(M,Ml),t(M,gs),t(gs,zl),t(M,Vl),t(M,De),$(St,De,null),t(De,Cl),t(De,vs),t(vs,jl),t(M,Ol),t(M,Y),$(At,Y,null),t(Y,Ql),t(Y,Lt),t(Lt,Ul),t(Lt,$s),t($s,Fl),t(Lt,Gl),t(Y,Rl),$(Te,Y,null),t(M,Hl),t(M,Ne),$(Dt,Ne,null),t(Ne,Wl),t(Ne,bs),t(bs,Xl),c(e,pa,p),c(e,Me,p),t(Me,Bl),t(Me,xs),t(xs,Jl),t(Me,Kl),c(e,ca,p),c(e,W,p),$(Tt,W,null),t(W,Yl),t(W,Es),t(Es,Zl),t(W,ei),$(ze,W,null),c(e,ma,p),c(e,Ve,p),t(Ve,ti),t(Ve,ws),t(ws,ri),t(Ve,si),c(e,fa,p),c(e,de,p),t(de,Ce),t(Ce,ys),$(Nt,ys,null),t(de,ai),t(de,Ps),t(Ps,oi),c(e,ua,p),c(e,dr,p),t(dr,ni),c(e,ha,p),$(Mt,e,p),c(e,_a,p),c(e,je,p),t(je,li),t(je,qs),t(qs,ii),t(je,di),c(e,ga,p),$(zt,e,p),c(e,va,p),c(e,Oe,p),t(Oe,pi),t(Oe,Vt),t(Vt,ci),t(Oe,mi),$a=!0},p(e,[p]){const Ct={};p&2&&(Ct.$$scope={dirty:p,ctx:e}),ce.$set(Ct);const ks={};p&2&&(ks.$$scope={dirty:p,ctx:e}),me.$set(ks);const Is={};p&2&&(Is.$$scope={dirty:p,ctx:e}),fe.$set(Is);const Ss={};p&2&&(Ss.$$scope={dirty:p,ctx:e}),ue.$set(Ss);const As={};p&2&&(As.$$scope={dirty:p,ctx:e}),Te.$set(As);const jt={};p&2&&(jt.$$scope={dirty:p,ctx:e}),ze.$set(jt)},i(e){$a||(b(f.$$.fragment,e),b(Ue.$$.fragment,e),b(Fe.$$.fragment,e),b(Ge.$$.fragment,e),b(ce.$$.fragment,e),b(Re.$$.fragment,e),b(me.$$.fragment,e),b(We.$$.fragment,e),b(fe.$$.fragment,e),b(Be.$$.fragment,e),b(ue.$$.fragment,e),b(Ke.$$.fragment,e),b(Ye.$$.fragment,e),b(Ze.$$.fragment,e),b(tt.$$.fragment,e),b(rt.$$.fragment,e),b(st.$$.fragment,e),b(ot.$$.fragment,e),b(lt.$$.fragment,e),b(it.$$.fragment,e),b(dt.$$.fragment,e),b(pt.$$.fragment,e),b(ct.$$.fragment,e),b(mt.$$.fragment,e),b(ht.$$.fragment,e),b(_t.$$.fragment,e),b(Et.$$.fragment,e),b(qt.$$.fragment,e),b(It.$$.fragment,e),b(St.$$.fragment,e),b(At.$$.fragment,e),b(Te.$$.fragment,e),b(Dt.$$.fragment,e),b(Tt.$$.fragment,e),b(ze.$$.fragment,e),b(Nt.$$.fragment,e),b(Mt.$$.fragment,e),b(zt.$$.fragment,e),$a=!0)},o(e){x(f.$$.fragment,e),x(Ue.$$.fragment,e),x(Fe.$$.fragment,e),x(Ge.$$.fragment,e),x(ce.$$.fragment,e),x(Re.$$.fragment,e),x(me.$$.fragment,e),x(We.$$.fragment,e),x(fe.$$.fragment,e),x(Be.$$.fragment,e),x(ue.$$.fragment,e),x(Ke.$$.fragment,e),x(Ye.$$.fragment,e),x(Ze.$$.fragment,e),x(tt.$$.fragment,e),x(rt.$$.fragment,e),x(st.$$.fragment,e),x(ot.$$.fragment,e),x(lt.$$.fragment,e),x(it.$$.fragment,e),x(dt.$$.fragment,e),x(pt.$$.fragment,e),x(ct.$$.fragment,e),x(mt.$$.fragment,e),x(ht.$$.fragment,e),x(_t.$$.fragment,e),x(Et.$$.fragment,e),x(qt.$$.fragment,e),x(It.$$.fragment,e),x(St.$$.fragment,e),x(At.$$.fragment,e),x(Te.$$.fragment,e),x(Dt.$$.fragment,e),x(Tt.$$.fragment,e),x(ze.$$.fragment,e),x(Nt.$$.fragment,e),x(Mt.$$.fragment,e),x(zt.$$.fragment,e),$a=!1},d(e){r(u),e&&r(y),e&&r(_),E(f),e&&r(A),e&&r(j),e&&r(N),e&&r(F),e&&r(Ts),e&&r(te),E(Ue),e&&r(Ns),e&&r(Gt),e&&r(Ms),e&&r(Rt),e&&r(zs),e&&r(L),E(Fe),E(Ge),E(ce),E(Re),E(me),E(We),E(fe),E(Be),E(ue),e&&r(Vs),e&&r(re),E(Ke),e&&r(Cs),e&&r(V),e&&r(js),e&&r(I),E(Ye),E(Ze),E(tt),E(rt),E(st),E(ot),E(lt),e&&r(Os),e&&r(G),E(it),E(dt),e&&r(Qs),e&&r(R),E(pt),E(ct),e&&r(Us),e&&r(se),E(mt),e&&r(Fs),e&&r(Pe),e&&r(Gs),e&&r(er),e&&r(Rs),e&&r(tr),e&&r(Hs),e&&r(q),e&&r(Ws),e&&r(qe),e&&r(Xs),e&&r(ae),E(ht),e&&r(Bs),e&&r(oe),E(_t),e&&r(Js),e&&r(ne),e&&r(Ks),e&&r($t),e&&r(Ys),e&&r(ar),e&&r(Zs),e&&r(or),e&&r(ea),e&&r(nr),e&&r(ta),e&&r(Ie),e&&r(ra),e&&r(le),E(Et),e&&r(sa),e&&r(H),e&&r(aa),e&&r(lr),e&&r(oa),e&&r(ie),E(qt),e&&r(na),e&&r(ir),e&&r(la),e&&r(Le),e&&r(ia),e&&r(kt),e&&r(da),e&&r(M),E(It),E(St),E(At),E(Te),E(Dt),e&&r(pa),e&&r(Me),e&&r(ca),e&&r(W),E(Tt),E(ze),e&&r(ma),e&&r(Ve),e&&r(fa),e&&r(de),E(Nt),e&&r(ua),e&&r(dr),e&&r(ha),E(Mt,e),e&&r(_a),e&&r(je),e&&r(ga),E(zt,e),e&&r(va),e&&r(Oe)}}}const wp={local:"processors",sections:[{local:"transformers.ProcessorMixin",title:"Multi-modal processors"},{local:"transformers.DataProcessor",title:"Deprecated processors"},{local:"transformers.glue_convert_examples_to_features",title:"GLUE"},{local:"xnli",title:"XNLI"},{local:"squad",sections:[{local:"transformers.data.processors.squad.SquadProcessor",title:"Processors"},{local:"example-usage",title:"Example usage"}],title:"SQuAD"}],title:"Processors"};function yp(z){return hp(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Lp extends cp{constructor(u){super();mp(this,u,yp,Ep,fp,{})}}export{Lp as default,wp as metadata};
