import{S as fr,i as ur,s as dr,e as r,k as h,w as f,t as n,M as mr,c as l,d as t,m as c,a as p,x as u,h as o,b as x,G as s,g as i,y as d,q as m,o as g,B as w,v as gr}from"../../chunks/vendor-hf-doc-builder.js";import{T as wr}from"../../chunks/Tip-hf-doc-builder.js";import{Y as xr}from"../../chunks/Youtube-hf-doc-builder.js";import{I as ba}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as k}from"../../chunks/CodeBlock-hf-doc-builder.js";import{C as _r}from"../../chunks/CourseFloatingBanner-hf-doc-builder.js";function kr(es){let b,R;return{c(){b=r("p"),R=n("\u26A0\uFE0F Training a tokenizer is not the same as training a model! Model training uses stochastic gradient descent to make the loss a little bit smaller for each batch. It\u2019s randomized by nature (meaning you have to set some seeds to get the same results when doing the same training twice). Training a tokenizer is a statistical process that tries to identify which subwords are the best to pick for a given corpus, and the exact rules used to pick them depend on the tokenization algorithm. It\u2019s deterministic, meaning you always get the same results when training with the same algorithm on the same corpus.")},l(j){b=l(j,"P",{});var z=p(b);R=o(z,"\u26A0\uFE0F Training a tokenizer is not the same as training a model! Model training uses stochastic gradient descent to make the loss a little bit smaller for each batch. It\u2019s randomized by nature (meaning you have to set some seeds to get the same results when doing the same training twice). Training a tokenizer is a statistical process that tries to identify which subwords are the best to pick for a given corpus, and the exact rules used to pick them depend on the tokenization algorithm. It\u2019s deterministic, meaning you always get the same results when training with the same algorithm on the same corpus."),z.forEach(t)},m(j,z){i(j,b,z),s(b,R)},d(j){j&&t(b)}}}function yr(es){let b,R,j,z,pt,ee,$a,ht,ja,ts,te,ss,E,va,Ne,za,Ea,ct,Ta,qa,ft,Pa,Aa,as,se,ns,U,os,N,H,ut,ae,Ca,dt,Da,is,W,Oa,mt,La,Sa,rs,v,Ia,ne,Na,Ga,gt,Ma,Ra,oe,Ua,Ha,ie,Wa,Ya,ls,re,ps,Ge,Ba,hs,le,cs,pe,fs,P,Fa,wt,Ka,Xa,xt,Ja,Qa,us,he,ds,Me,Za,ms,ce,gs,Y,Va,_t,en,tn,ws,Re,sn,xs,fe,_s,Ue,an,ks,ue,ys,A,nn,kt,on,rn,yt,ln,pn,bs,He,hn,$s,de,js,We,cn,vs,me,zs,Ye,fn,Es,ge,Ts,C,un,bt,dn,mn,$t,gn,wn,qs,we,Ps,Be,xn,As,G,B,jt,xe,_n,vt,kn,Cs,Fe,yn,Ds,_e,Os,Ke,bn,Ls,Xe,$n,Ss,ke,Is,ye,Ns,T,jn,zt,vn,zn,Et,En,Tn,Tt,qn,Pn,Gs,F,An,qt,Cn,Dn,Ms,be,Rs,Je,On,Us,D,Ln,Pt,Sn,In,$e,Nn,Gn,Hs,K,Mn,Qe,Rn,Un,Ws,O,Hn,je,Wn,Yn,At,Bn,Fn,Ys,ve,Bs,ze,Fs,$,Kn,Ct,Xn,Jn,Dt,Qn,Zn,Ot,Vn,eo,Lt,to,so,St,ao,no,Ks,Ee,Xs,Te,Js,Ze,oo,Qs,qe,Zs,Pe,Vs,_,io,It,ro,lo,Nt,po,ho,Gt,co,fo,Mt,uo,mo,Rt,go,wo,Ut,xo,_o,Ht,ko,yo,Wt,bo,$o,Yt,jo,vo,Bt,zo,Eo,ea,M,X,Ft,Ae,To,Kt,qo,ta,J,Po,Xt,Ao,Co,sa,Ce,aa,Q,Do,Jt,Oo,Lo,na,De,oa,Ve,So,ia,Oe,ra,et,Io,la,Le,pa,L,No,Qt,Go,Mo,Zt,Ro,Uo,ha,Se,ca,S,Ho,tt,Wo,Yo,Vt,Bo,Fo,fa;return ee=new ba({}),te=new _r({props:{chapter:6,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter6/section2.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter6/section2.ipynb"}]}}),se=new xr({props:{id:"DJimQynXZsQ"}}),U=new wr({props:{warning:!0,$$slots:{default:[kr]},$$scope:{ctx:es}}}),ae=new ba({}),re=new k({props:{code:`from datasets import load_dataset

# This can take a few minutes to load, so grab a coffee or tea while you wait!
raw_datasets = load_dataset("code_search_net", "python")`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-comment"># This can take a few minutes to load, so grab a coffee or tea while you wait!</span>
raw_datasets = load_dataset(<span class="hljs-string">&quot;code_search_net&quot;</span>, <span class="hljs-string">&quot;python&quot;</span>)`}}),le=new k({props:{code:'raw_datasets["train"]',highlighted:'raw_datasets[<span class="hljs-string">&quot;train&quot;</span>]'}}),pe=new k({props:{code:`Dataset({
    features: ['repository_name', 'func_path_in_repository', 'func_name', 'whole_func_string', 'language', 
      'func_code_string', 'func_code_tokens', 'func_documentation_string', 'func_documentation_tokens', 'split_name', 
      'func_code_url'
    ],
    num_rows: 412178
})`,highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;repository_name&#x27;</span>, <span class="hljs-string">&#x27;func_path_in_repository&#x27;</span>, <span class="hljs-string">&#x27;func_name&#x27;</span>, <span class="hljs-string">&#x27;whole_func_string&#x27;</span>, <span class="hljs-string">&#x27;language&#x27;</span>, 
      <span class="hljs-string">&#x27;func_code_string&#x27;</span>, <span class="hljs-string">&#x27;func_code_tokens&#x27;</span>, <span class="hljs-string">&#x27;func_documentation_string&#x27;</span>, <span class="hljs-string">&#x27;func_documentation_tokens&#x27;</span>, <span class="hljs-string">&#x27;split_name&#x27;</span>, 
      <span class="hljs-string">&#x27;func_code_url&#x27;</span>
    ],
    num_rows: <span class="hljs-number">412178</span>
})`}}),he=new k({props:{code:'print(raw_datasets["train"][123456]["whole_func_string"])',highlighted:'<span class="hljs-built_in">print</span>(raw_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">123456</span>][<span class="hljs-string">&quot;whole_func_string&quot;</span>])'}}),ce=new k({props:{code:`def handle_simple_responses(
      self, timeout_ms=None, info_cb=DEFAULT_MESSAGE_CALLBACK):
    """Accepts normal responses from the device.

    Args:
      timeout_ms: Timeout in milliseconds to wait for each response.
      info_cb: Optional callback for text sent from the bootloader.

    Returns:
      OKAY packet's message.
    """
    return self._accept_responses('OKAY', info_cb, timeout_ms=timeout_ms)`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">handle_simple_responses</span>(<span class="hljs-params">
      self, timeout_ms=<span class="hljs-literal">None</span>, info_cb=DEFAULT_MESSAGE_CALLBACK</span>):
    <span class="hljs-string">&quot;&quot;&quot;Accepts normal responses from the device.

    Args:
      timeout_ms: Timeout in milliseconds to wait for each response.
      info_cb: Optional callback for text sent from the bootloader.

    Returns:
      OKAY packet&#x27;s message.
    &quot;&quot;&quot;</span>
    <span class="hljs-keyword">return</span> self._accept_responses(<span class="hljs-string">&#x27;OKAY&#x27;</span>, info_cb, timeout_ms=timeout_ms)`}}),fe=new k({props:{code:`# Don't uncomment the following line unless your dataset is small!
# training_corpus = [raw_datasets["train"][i: i + 1000]["whole_func_string"] for i in range(0, len(raw_datasets["train"]), 1000)]`,highlighted:`<span class="hljs-comment"># Don&#x27;t uncomment the following line unless your dataset is small!</span>
<span class="hljs-comment"># training_corpus = [raw_datasets[&quot;train&quot;][i: i + 1000][&quot;whole_func_string&quot;] for i in range(0, len(raw_datasets[&quot;train&quot;]), 1000)]</span>`}}),ue=new k({props:{code:`training_corpus = (
    raw_datasets["train"][i : i + 1000]["whole_func_string"]
    for i in range(0, len(raw_datasets["train"]), 1000)
)`,highlighted:`training_corpus = (
    raw_datasets[<span class="hljs-string">&quot;train&quot;</span>][i : i + <span class="hljs-number">1000</span>][<span class="hljs-string">&quot;whole_func_string&quot;</span>]
    <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">0</span>, <span class="hljs-built_in">len</span>(raw_datasets[<span class="hljs-string">&quot;train&quot;</span>]), <span class="hljs-number">1000</span>)
)`}}),de=new k({props:{code:`gen = (i for i in range(10))
print(list(gen))
print(list(gen))`,highlighted:`gen = (i <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">10</span>))
<span class="hljs-built_in">print</span>(<span class="hljs-built_in">list</span>(gen))
<span class="hljs-built_in">print</span>(<span class="hljs-built_in">list</span>(gen))`}}),me=new k({props:{code:`[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
[]`,highlighted:`[<span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3</span>, <span class="hljs-number">4</span>, <span class="hljs-number">5</span>, <span class="hljs-number">6</span>, <span class="hljs-number">7</span>, <span class="hljs-number">8</span>, <span class="hljs-number">9</span>]
[]`}}),ge=new k({props:{code:`def get_training_corpus():
    return (
        raw_datasets["train"][i : i + 1000]["whole_func_string"]
        for i in range(0, len(raw_datasets["train"]), 1000)
    )


training_corpus = get_training_corpus()`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_training_corpus</span>():
    <span class="hljs-keyword">return</span> (
        raw_datasets[<span class="hljs-string">&quot;train&quot;</span>][i : i + <span class="hljs-number">1000</span>][<span class="hljs-string">&quot;whole_func_string&quot;</span>]
        <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">0</span>, <span class="hljs-built_in">len</span>(raw_datasets[<span class="hljs-string">&quot;train&quot;</span>]), <span class="hljs-number">1000</span>)
    )


training_corpus = get_training_corpus()`}}),we=new k({props:{code:`def get_training_corpus():
    dataset = raw_datasets["train"]
    for start_idx in range(0, len(dataset), 1000):
        samples = dataset[start_idx : start_idx + 1000]
        yield samples["whole_func_string"]`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_training_corpus</span>():
    dataset = raw_datasets[<span class="hljs-string">&quot;train&quot;</span>]
    <span class="hljs-keyword">for</span> start_idx <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">0</span>, <span class="hljs-built_in">len</span>(dataset), <span class="hljs-number">1000</span>):
        samples = dataset[start_idx : start_idx + <span class="hljs-number">1000</span>]
        <span class="hljs-keyword">yield</span> samples[<span class="hljs-string">&quot;whole_func_string&quot;</span>]`}}),xe=new ba({}),_e=new k({props:{code:`from transformers import AutoTokenizer

old_tokenizer = AutoTokenizer.from_pretrained("gpt2")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

old_tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;gpt2&quot;</span>)`}}),ke=new k({props:{code:`example = '''def add_numbers(a, b):
    """Add the two numbers \`a\` and \`b\`."""
    return a + b'''

tokens = old_tokenizer.tokenize(example)
tokens`,highlighted:`example = <span class="hljs-string">&#x27;&#x27;&#x27;def add_numbers(a, b):
    &quot;&quot;&quot;Add the two numbers \`a\` and \`b\`.&quot;&quot;&quot;
    return a + b&#x27;&#x27;&#x27;</span>

tokens = old_tokenizer.tokenize(example)
tokens`}}),ye=new k({props:{code:`['def', '\u0120add', '_', 'n', 'umbers', '(', 'a', ',', '\u0120b', '):', '\u010A', '\u0120', '\u0120', '\u0120', '\u0120"""', 'Add', '\u0120the', '\u0120two',
 '\u0120numbers', '\u0120\`', 'a', '\`', '\u0120and', '\u0120\`', 'b', '\`', '."', '""', '\u010A', '\u0120', '\u0120', '\u0120', '\u0120return', '\u0120a', '\u0120+', '\u0120b']`,highlighted:'[<span class="hljs-string">&#x27;def&#x27;</span>, <span class="hljs-string">&#x27;\u0120add&#x27;</span>, <span class="hljs-string">&#x27;_&#x27;</span>, <span class="hljs-string">&#x27;n&#x27;</span>, <span class="hljs-string">&#x27;umbers&#x27;</span>, <span class="hljs-string">&#x27;(&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;\u0120b&#x27;</span>, <span class="hljs-string">&#x27;):&#x27;</span>, <span class="hljs-string">&#x27;\u010A&#x27;</span>, <span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120&quot;&quot;&quot;&#x27;</span>, <span class="hljs-string">&#x27;Add&#x27;</span>, <span class="hljs-string">&#x27;\u0120the&#x27;</span>, <span class="hljs-string">&#x27;\u0120two&#x27;</span>,\n <span class="hljs-string">&#x27;\u0120numbers&#x27;</span>, <span class="hljs-string">&#x27;\u0120`&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;`&#x27;</span>, <span class="hljs-string">&#x27;\u0120and&#x27;</span>, <span class="hljs-string">&#x27;\u0120`&#x27;</span>, <span class="hljs-string">&#x27;b&#x27;</span>, <span class="hljs-string">&#x27;`&#x27;</span>, <span class="hljs-string">&#x27;.&quot;&#x27;</span>, <span class="hljs-string">&#x27;&quot;&quot;&#x27;</span>, <span class="hljs-string">&#x27;\u010A&#x27;</span>, <span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120return&#x27;</span>, <span class="hljs-string">&#x27;\u0120a&#x27;</span>, <span class="hljs-string">&#x27;\u0120+&#x27;</span>, <span class="hljs-string">&#x27;\u0120b&#x27;</span>]'}}),be=new k({props:{code:"tokenizer = old_tokenizer.train_new_from_iterator(training_corpus, 52000)",highlighted:'tokenizer = old_tokenizer.train_new_from_iterator(training_corpus, <span class="hljs-number">52000</span>)'}}),ve=new k({props:{code:`tokens = tokenizer.tokenize(example)
tokens`,highlighted:`tokens = tokenizer.tokenize(example)
tokens`}}),ze=new k({props:{code:`['def', '\u0120add', '_', 'numbers', '(', 'a', ',', '\u0120b', '):', '\u010A\u0120\u0120\u0120', '\u0120"""', 'Add', '\u0120the', '\u0120two', '\u0120numbers', '\u0120\`',
 'a', '\`', '\u0120and', '\u0120\`', 'b', '\`."""', '\u010A\u0120\u0120\u0120', '\u0120return', '\u0120a', '\u0120+', '\u0120b']`,highlighted:'[<span class="hljs-string">&#x27;def&#x27;</span>, <span class="hljs-string">&#x27;\u0120add&#x27;</span>, <span class="hljs-string">&#x27;_&#x27;</span>, <span class="hljs-string">&#x27;numbers&#x27;</span>, <span class="hljs-string">&#x27;(&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;\u0120b&#x27;</span>, <span class="hljs-string">&#x27;):&#x27;</span>, <span class="hljs-string">&#x27;\u010A\u0120\u0120\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120&quot;&quot;&quot;&#x27;</span>, <span class="hljs-string">&#x27;Add&#x27;</span>, <span class="hljs-string">&#x27;\u0120the&#x27;</span>, <span class="hljs-string">&#x27;\u0120two&#x27;</span>, <span class="hljs-string">&#x27;\u0120numbers&#x27;</span>, <span class="hljs-string">&#x27;\u0120`&#x27;</span>,\n <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;`&#x27;</span>, <span class="hljs-string">&#x27;\u0120and&#x27;</span>, <span class="hljs-string">&#x27;\u0120`&#x27;</span>, <span class="hljs-string">&#x27;b&#x27;</span>, <span class="hljs-string">&#x27;`.&quot;&quot;&quot;&#x27;</span>, <span class="hljs-string">&#x27;\u010A\u0120\u0120\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120return&#x27;</span>, <span class="hljs-string">&#x27;\u0120a&#x27;</span>, <span class="hljs-string">&#x27;\u0120+&#x27;</span>, <span class="hljs-string">&#x27;\u0120b&#x27;</span>]'}}),Ee=new k({props:{code:`print(len(tokens))
print(len(old_tokenizer.tokenize(example)))`,highlighted:`<span class="hljs-built_in">print</span>(<span class="hljs-built_in">len</span>(tokens))
<span class="hljs-built_in">print</span>(<span class="hljs-built_in">len</span>(old_tokenizer.tokenize(example)))`}}),Te=new k({props:{code:`27
36`,highlighted:`<span class="hljs-number">27</span>
<span class="hljs-number">36</span>`}}),qe=new k({props:{code:`example = """class LinearLayer():
    def __init__(self, input_size, output_size):
        self.weight = torch.randn(input_size, output_size)
        self.bias = torch.zeros(output_size)

    def __call__(self, x):
        return x @ self.weights + self.bias
    """
tokenizer.tokenize(example)`,highlighted:`example = <span class="hljs-string">&quot;&quot;&quot;class LinearLayer():
    def __init__(self, input_size, output_size):
        self.weight = torch.randn(input_size, output_size)
        self.bias = torch.zeros(output_size)

    def __call__(self, x):
        return x @ self.weights + self.bias
    &quot;&quot;&quot;</span>
tokenizer.tokenize(example)`}}),Pe=new k({props:{code:`['class', '\u0120Linear', 'Layer', '():', '\u010A\u0120\u0120\u0120', '\u0120def', '\u0120__', 'init', '__(', 'self', ',', '\u0120input', '_', 'size', ',',
 '\u0120output', '_', 'size', '):', '\u010A\u0120\u0120\u0120\u0120\u0120\u0120\u0120', '\u0120self', '.', 'weight', '\u0120=', '\u0120torch', '.', 'randn', '(', 'input', '_',
 'size', ',', '\u0120output', '_', 'size', ')', '\u010A\u0120\u0120\u0120\u0120\u0120\u0120\u0120', '\u0120self', '.', 'bias', '\u0120=', '\u0120torch', '.', 'zeros', '(',
 'output', '_', 'size', ')', '\u010A\u010A\u0120\u0120\u0120', '\u0120def', '\u0120__', 'call', '__(', 'self', ',', '\u0120x', '):', '\u010A\u0120\u0120\u0120\u0120\u0120\u0120\u0120',
 '\u0120return', '\u0120x', '\u0120@', '\u0120self', '.', 'weights', '\u0120+', '\u0120self', '.', 'bias', '\u010A\u0120\u0120\u0120\u0120']`,highlighted:`[<span class="hljs-string">&#x27;class&#x27;</span>, <span class="hljs-string">&#x27;\u0120Linear&#x27;</span>, <span class="hljs-string">&#x27;Layer&#x27;</span>, <span class="hljs-string">&#x27;():&#x27;</span>, <span class="hljs-string">&#x27;\u010A\u0120\u0120\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120def&#x27;</span>, <span class="hljs-string">&#x27;\u0120__&#x27;</span>, <span class="hljs-string">&#x27;init&#x27;</span>, <span class="hljs-string">&#x27;__(&#x27;</span>, <span class="hljs-string">&#x27;self&#x27;</span>, <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;\u0120input&#x27;</span>, <span class="hljs-string">&#x27;_&#x27;</span>, <span class="hljs-string">&#x27;size&#x27;</span>, <span class="hljs-string">&#x27;,&#x27;</span>,
 <span class="hljs-string">&#x27;\u0120output&#x27;</span>, <span class="hljs-string">&#x27;_&#x27;</span>, <span class="hljs-string">&#x27;size&#x27;</span>, <span class="hljs-string">&#x27;):&#x27;</span>, <span class="hljs-string">&#x27;\u010A\u0120\u0120\u0120\u0120\u0120\u0120\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120self&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;weight&#x27;</span>, <span class="hljs-string">&#x27;\u0120=&#x27;</span>, <span class="hljs-string">&#x27;\u0120torch&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;randn&#x27;</span>, <span class="hljs-string">&#x27;(&#x27;</span>, <span class="hljs-string">&#x27;input&#x27;</span>, <span class="hljs-string">&#x27;_&#x27;</span>,
 <span class="hljs-string">&#x27;size&#x27;</span>, <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;\u0120output&#x27;</span>, <span class="hljs-string">&#x27;_&#x27;</span>, <span class="hljs-string">&#x27;size&#x27;</span>, <span class="hljs-string">&#x27;)&#x27;</span>, <span class="hljs-string">&#x27;\u010A\u0120\u0120\u0120\u0120\u0120\u0120\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120self&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;bias&#x27;</span>, <span class="hljs-string">&#x27;\u0120=&#x27;</span>, <span class="hljs-string">&#x27;\u0120torch&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;zeros&#x27;</span>, <span class="hljs-string">&#x27;(&#x27;</span>,
 <span class="hljs-string">&#x27;output&#x27;</span>, <span class="hljs-string">&#x27;_&#x27;</span>, <span class="hljs-string">&#x27;size&#x27;</span>, <span class="hljs-string">&#x27;)&#x27;</span>, <span class="hljs-string">&#x27;\u010A\u010A\u0120\u0120\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120def&#x27;</span>, <span class="hljs-string">&#x27;\u0120__&#x27;</span>, <span class="hljs-string">&#x27;call&#x27;</span>, <span class="hljs-string">&#x27;__(&#x27;</span>, <span class="hljs-string">&#x27;self&#x27;</span>, <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;\u0120x&#x27;</span>, <span class="hljs-string">&#x27;):&#x27;</span>, <span class="hljs-string">&#x27;\u010A\u0120\u0120\u0120\u0120\u0120\u0120\u0120&#x27;</span>,
 <span class="hljs-string">&#x27;\u0120return&#x27;</span>, <span class="hljs-string">&#x27;\u0120x&#x27;</span>, <span class="hljs-string">&#x27;\u0120@&#x27;</span>, <span class="hljs-string">&#x27;\u0120self&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;weights&#x27;</span>, <span class="hljs-string">&#x27;\u0120+&#x27;</span>, <span class="hljs-string">&#x27;\u0120self&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;bias&#x27;</span>, <span class="hljs-string">&#x27;\u010A\u0120\u0120\u0120\u0120&#x27;</span>]`}}),Ae=new ba({}),Ce=new k({props:{code:'tokenizer.save_pretrained("code-search-net-tokenizer")',highlighted:'tokenizer.save_pretrained(<span class="hljs-string">&quot;code-search-net-tokenizer&quot;</span>)'}}),De=new k({props:{code:`from huggingface_hub import notebook_login

notebook_login()`,highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

notebook_login()`}}),Oe=new k({props:{code:"huggingface-cli login",highlighted:"huggingface-cli login"}}),Le=new k({props:{code:'tokenizer.push_to_hub("code-search-net-tokenizer")',highlighted:'tokenizer.push_to_hub(<span class="hljs-string">&quot;code-search-net-tokenizer&quot;</span>)'}}),Se=new k({props:{code:`# Replace "huggingface-course" below with your actual namespace to use your own tokenizer
tokenizer = AutoTokenizer.from_pretrained("huggingface-course/code-search-net-tokenizer")`,highlighted:`<span class="hljs-comment"># Replace &quot;huggingface-course&quot; below with your actual namespace to use your own tokenizer</span>
tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;huggingface-course/code-search-net-tokenizer&quot;</span>)`}}),{c(){b=r("meta"),R=h(),j=r("h1"),z=r("a"),pt=r("span"),f(ee.$$.fragment),$a=h(),ht=r("span"),ja=n("Training a new tokenizer from an old one"),ts=h(),f(te.$$.fragment),ss=h(),E=r("p"),va=n("If a language model is not available in the language you are interested in, or if your corpus is very different from the one your language model was trained on, you will most likely want to retrain the model from scratch using a tokenizer adapted to your data. That will require training a new tokenizer on your dataset. But what exactly does that mean? When we first looked at tokenizers in "),Ne=r("a"),za=n("Chapter 2"),Ea=n(", we saw that most Transformer models use a "),ct=r("em"),Ta=n("subword tokenization algorithm"),qa=n(". To identify which subwords are of interest and occur most frequently in the corpus at hand, the tokenizer needs to take a hard look at all the texts in the corpus \u2014 a process we call "),ft=r("em"),Pa=n("training"),Aa=n(". The exact rules that govern this training depend on the type of tokenizer used, and we\u2019ll go over the three main algorithms later in this chapter."),as=h(),f(se.$$.fragment),ns=h(),f(U.$$.fragment),os=h(),N=r("h2"),H=r("a"),ut=r("span"),f(ae.$$.fragment),Ca=h(),dt=r("span"),Da=n("Assembling a corpus"),is=h(),W=r("p"),Oa=n("There\u2019s a very simple API in \u{1F917} Transformers that you can use to train a new tokenizer with the same characteristics as an existing one: "),mt=r("code"),La=n("AutoTokenizer.train_new_from_iterator()"),Sa=n(". To see this in action, let\u2019s say we want to train GPT-2 from scratch, but in a language other than English. Our first task will be to gather lots of data in that language in a training corpus. To provide examples everyone will be able to understand, we won\u2019t use a language like Russian or Chinese here, but rather a specialized English language: Python code."),rs=h(),v=r("p"),Ia=n("The "),ne=r("a"),Na=n("\u{1F917} Datasets"),Ga=n(" library can help us assemble a corpus of Python source code. We\u2019ll use the usual "),gt=r("code"),Ma=n("load_dataset()"),Ra=n(" function to download and cache the "),oe=r("a"),Ua=n("CodeSearchNet"),Ha=n(" dataset. This dataset was created for the "),ie=r("a"),Wa=n("CodeSearchNet challenge"),Ya=n(" and contains millions of functions from open source libraries on GitHub in several programming languages. Here, we will load the Python part of this dataset:"),ls=h(),f(re.$$.fragment),ps=h(),Ge=r("p"),Ba=n("We can have a look at the training split to see which columns we have access to:"),hs=h(),f(le.$$.fragment),cs=h(),f(pe.$$.fragment),fs=h(),P=r("p"),Fa=n("We can see the dataset separates docstrings from code and suggests a tokenization of both. Here. we\u2019ll just use the "),wt=r("code"),Ka=n("whole_func_string"),Xa=n(" column to train our tokenizer. We can look at an example of one these functions by indexing into the "),xt=r("code"),Ja=n("train"),Qa=n(" split:"),us=h(),f(he.$$.fragment),ds=h(),Me=r("p"),Za=n("which should print the following:"),ms=h(),f(ce.$$.fragment),gs=h(),Y=r("p"),Va=n("The first thing we need to do is transform the dataset into an "),_t=r("em"),en=n("iterator"),tn=n(" of lists of texts \u2014 for instance, a list of list of texts. Using lists of texts will enable our tokenizer to go faster (training on batches of texts instead of processing individual texts one by one), and it should be an iterator if we want to avoid having everything in memory at once. If your corpus is huge, you will want to take advantage of the fact that \u{1F917} Datasets does not load everything into RAM but stores the elements of the dataset on disk."),ws=h(),Re=r("p"),sn=n("Doing the following would create a list of lists of 1,000 texts each, but would load everything in memory:"),xs=h(),f(fe.$$.fragment),_s=h(),Ue=r("p"),an=n("Using a Python generator, we can avoid Python loading anything into memory until it\u2019s actually necessary. To create such a generator, you just to need to replace the brackets with parentheses:"),ks=h(),f(ue.$$.fragment),ys=h(),A=r("p"),nn=n("This line of code doesn\u2019t fetch any elements of the dataset; it just creates an object you can use in a Python "),kt=r("code"),on=n("for"),rn=n(" loop. The texts will only be loaded when you need them (that is, when you\u2019re at the step of the "),yt=r("code"),ln=n("for"),pn=n(" loop that requires them), and only 1,000 texts at a time will be loaded. This way you won\u2019t exhaust all your memory even if you are processing a huge dataset."),bs=h(),He=r("p"),hn=n("The problem with a generator object is that it can only be used once. So, instead of this giving us the list of the first 10 digits twice:"),$s=h(),f(de.$$.fragment),js=h(),We=r("p"),cn=n("we get them once and then an empty list:"),vs=h(),f(me.$$.fragment),zs=h(),Ye=r("p"),fn=n("That\u2019s why we define a function that returns a generator instead:"),Es=h(),f(ge.$$.fragment),Ts=h(),C=r("p"),un=n("You can also define your generator inside a "),bt=r("code"),dn=n("for"),mn=n(" loop by using the "),$t=r("code"),gn=n("yield"),wn=n(" statement:"),qs=h(),f(we.$$.fragment),Ps=h(),Be=r("p"),xn=n("which will produce the exact same generator as before, but allows you to use more complex logic than you can in a list comprehension."),As=h(),G=r("h2"),B=r("a"),jt=r("span"),f(xe.$$.fragment),_n=h(),vt=r("span"),kn=n("Training a new tokenizer"),Cs=h(),Fe=r("p"),yn=n("Now that we have our corpus in the form of an iterator of batches of texts, we are ready to train a new tokenizer. To do this, we first need to load the tokenizer we want to pair with our model (here, GPT-2):"),Ds=h(),f(_e.$$.fragment),Os=h(),Ke=r("p"),bn=n("Even though we are going to train a new tokenizer, it\u2019s a good idea to do this to avoid starting entirely from scratch. This way, we won\u2019t have to specify anything about the tokenization algorithm or the special tokens we want to use; our new tokenizer will be exactly the same as GPT-2, and the only thing that will change is the vocabulary, which will be determined by the training on our corpus."),Ls=h(),Xe=r("p"),$n=n("First let\u2019s have a look at how this tokenizer would treat an example function:"),Ss=h(),f(ke.$$.fragment),Is=h(),f(ye.$$.fragment),Ns=h(),T=r("p"),jn=n("This tokenizer has a few special symbols, like "),zt=r("code"),vn=n("\u0120"),zn=n(" and "),Et=r("code"),En=n("\u010A"),Tn=n(", which denote spaces and newlines, respectively. As we can see, this is not too efficient: the tokenizer returns individual tokens for each space, when it could group together indentation levels (since having sets of four or eight spaces is going to be very common in code). It also split the function name a bit weirdly, not being used to seeing words with the "),Tt=r("code"),qn=n("_"),Pn=n(" character."),Gs=h(),F=r("p"),An=n("Let\u2019s train a new tokenizer and see if it solves those issues. For this, we\u2019ll use the method "),qt=r("code"),Cn=n("train_new_from_iterator()"),Dn=n(":"),Ms=h(),f(be.$$.fragment),Rs=h(),Je=r("p"),On=n("This command might take a bit of time if your corpus is very large, but for this dataset of 1.6 GB of texts it\u2019s  blazing fast (1 minute 16 seconds on an AMD Ryzen 9 3900X CPU with 12 cores)."),Us=h(),D=r("p"),Ln=n("Note that "),Pt=r("code"),Sn=n("AutoTokenizer.train_new_from_iterator()"),In=n(" only works if the tokenizer you are using is a \u201Cfast\u201D tokenizer. As you\u2019ll see in the next section, the \u{1F917} Transformers library contains two types of tokenizers: some are written purely in Python and others (the fast ones) are backed by the \u{1F917} Tokenizers library, which is written in the "),$e=r("a"),Nn=n("Rust"),Gn=n(" programming language. Python is the language most often used for data science and deep learning applications, but when anything needs to be parallelized to be fast, it has to be written in another language. For instance, the matrix multiplications that are at the core of the model computation are written in CUDA, an optimized C library for GPUs."),Hs=h(),K=r("p"),Mn=n("Training a brand new tokenizer in pure Python would be excruciatingly slow, which is why we developed the \u{1F917} Tokenizers library. Note that just as you didn\u2019t have to learn the CUDA language to be able to execute your model on a batch of inputs on a GPU, you won\u2019t need to learn Rust to use a fast tokenizer. The \u{1F917} Tokenizers library provides Python bindings for many methods that internally call some piece of code in Rust; for example, to parallelize the training of your new tokenizer or, as we saw in "),Qe=r("a"),Rn=n("Chapter 3"),Un=n(", the tokenization of a batch of inputs."),Ws=h(),O=r("p"),Hn=n("Most of the Transformer models have a fast tokenizer available (there are some exceptions that you can check "),je=r("a"),Wn=n("here"),Yn=n("), and the "),At=r("code"),Bn=n("AutoTokenizer"),Fn=n(" API always selects the fast tokenizer for you if it\u2019s available. In the next section we\u2019ll take a look at some of the other special features fast tokenizers have, which will be really useful for tasks like token classification and question answering. Before diving into that, however, let\u2019s try our brand new tokenizer on the previous example:"),Ys=h(),f(ve.$$.fragment),Bs=h(),f(ze.$$.fragment),Fs=h(),$=r("p"),Kn=n("Here we again see the special symbols "),Ct=r("code"),Xn=n("\u0120"),Jn=n(" and "),Dt=r("code"),Qn=n("\u010A"),Zn=n(" that denote spaces and newlines, but we can also see that our tokenizer learned some tokens that are highly specific to a corpus of Python functions: for example, there is a "),Ot=r("code"),Vn=n("\u010A\u0120\u0120\u0120"),eo=n(" token that represents an indentation, and a "),Lt=r("code"),to=n('\u0120"""'),so=n(" token that represents the three quotes that start a docstring. The tokenizer also correctly split the function name on "),St=r("code"),ao=n("_"),no=n(". This is quite a compact representation; comparatively, using the plain English tokenizer on the same example will give us a longer sentence:"),Ks=h(),f(Ee.$$.fragment),Xs=h(),f(Te.$$.fragment),Js=h(),Ze=r("p"),oo=n("Let\u2019s look at another example:"),Qs=h(),f(qe.$$.fragment),Zs=h(),f(Pe.$$.fragment),Vs=h(),_=r("p"),io=n("In addition to the token corresponding to an indentation, here we can also see a token for a double indentation: "),It=r("code"),ro=n("\u010A\u0120\u0120\u0120\u0120\u0120\u0120\u0120"),lo=n(". The special Python words like "),Nt=r("code"),po=n("class"),ho=n(", "),Gt=r("code"),co=n("init"),fo=n(", "),Mt=r("code"),uo=n("call"),mo=n(", "),Rt=r("code"),go=n("self"),wo=n(", and "),Ut=r("code"),xo=n("return"),_o=n(" are each tokenized as one token, and we can see that as well as splitting on "),Ht=r("code"),ko=n("_"),yo=n(" and "),Wt=r("code"),bo=n("."),$o=n(" the tokenizer correctly splits even camel-cased names: "),Yt=r("code"),jo=n("LinearLayer"),vo=n(" is tokenized as "),Bt=r("code"),zo=n('["\u0120Linear", "Layer"]'),Eo=n("."),ea=h(),M=r("h2"),X=r("a"),Ft=r("span"),f(Ae.$$.fragment),To=h(),Kt=r("span"),qo=n("Saving the tokenizer"),ta=h(),J=r("p"),Po=n("To make sure we can use it later, we need to save our new tokenizer. Like for models, this is done with the "),Xt=r("code"),Ao=n("save_pretrained()"),Co=n(" method:"),sa=h(),f(Ce.$$.fragment),aa=h(),Q=r("p"),Do=n("This will create a new folder named "),Jt=r("em"),Oo=n("code-search-net-tokenizer"),Lo=n(", which will contain all the files the tokenizer needs to be reloaded. If you want to share this tokenizer with your colleagues and friends, you can upload it to the Hub by logging into your account. If you\u2019re working in a notebook, there\u2019s a convenience function to help you with this:"),na=h(),f(De.$$.fragment),oa=h(),Ve=r("p"),So=n("This will display a widget where you can enter your Hugging Face login credentials. If you aren\u2019t working in a notebook, just type the following line in your terminal:"),ia=h(),f(Oe.$$.fragment),ra=h(),et=r("p"),Io=n("Once you\u2019ve logged in, you can push your tokenizer by executing the following command:"),la=h(),f(Le.$$.fragment),pa=h(),L=r("p"),No=n("This will create a new repository in your namespace with the name "),Qt=r("code"),Go=n("code-search-net-tokenizer"),Mo=n(", containing the tokenizer file. You can then load the tokenizer from anywhere with the "),Zt=r("code"),Ro=n("from_pretrained()"),Uo=n(" method:"),ha=h(),f(Se.$$.fragment),ca=h(),S=r("p"),Ho=n("You\u2019re now all set for training a language model from scratch and fine-tuning it on your task at hand! We\u2019ll get to that in "),tt=r("a"),Wo=n("Chapter 7"),Yo=n(", but first, in the rest of this chapter we\u2019ll take a closer look at fast tokenizers and explore in detail what actually happens when we call the method "),Vt=r("code"),Bo=n("train_new_from_iterator()"),Fo=n("."),this.h()},l(e){const a=mr('[data-svelte="svelte-1phssyn"]',document.head);b=l(a,"META",{name:!0,content:!0}),a.forEach(t),R=c(e),j=l(e,"H1",{class:!0});var Ie=p(j);z=l(Ie,"A",{id:!0,class:!0,href:!0});var Ko=p(z);pt=l(Ko,"SPAN",{});var Xo=p(pt);u(ee.$$.fragment,Xo),Xo.forEach(t),Ko.forEach(t),$a=c(Ie),ht=l(Ie,"SPAN",{});var Jo=p(ht);ja=o(Jo,"Training a new tokenizer from an old one"),Jo.forEach(t),Ie.forEach(t),ts=c(e),u(te.$$.fragment,e),ss=c(e),E=l(e,"P",{});var Z=p(E);va=o(Z,"If a language model is not available in the language you are interested in, or if your corpus is very different from the one your language model was trained on, you will most likely want to retrain the model from scratch using a tokenizer adapted to your data. That will require training a new tokenizer on your dataset. But what exactly does that mean? When we first looked at tokenizers in "),Ne=l(Z,"A",{href:!0});var Qo=p(Ne);za=o(Qo,"Chapter 2"),Qo.forEach(t),Ea=o(Z,", we saw that most Transformer models use a "),ct=l(Z,"EM",{});var Zo=p(ct);Ta=o(Zo,"subword tokenization algorithm"),Zo.forEach(t),qa=o(Z,". To identify which subwords are of interest and occur most frequently in the corpus at hand, the tokenizer needs to take a hard look at all the texts in the corpus \u2014 a process we call "),ft=l(Z,"EM",{});var Vo=p(ft);Pa=o(Vo,"training"),Vo.forEach(t),Aa=o(Z,". The exact rules that govern this training depend on the type of tokenizer used, and we\u2019ll go over the three main algorithms later in this chapter."),Z.forEach(t),as=c(e),u(se.$$.fragment,e),ns=c(e),u(U.$$.fragment,e),os=c(e),N=l(e,"H2",{class:!0});var ua=p(N);H=l(ua,"A",{id:!0,class:!0,href:!0});var ei=p(H);ut=l(ei,"SPAN",{});var ti=p(ut);u(ae.$$.fragment,ti),ti.forEach(t),ei.forEach(t),Ca=c(ua),dt=l(ua,"SPAN",{});var si=p(dt);Da=o(si,"Assembling a corpus"),si.forEach(t),ua.forEach(t),is=c(e),W=l(e,"P",{});var da=p(W);Oa=o(da,"There\u2019s a very simple API in \u{1F917} Transformers that you can use to train a new tokenizer with the same characteristics as an existing one: "),mt=l(da,"CODE",{});var ai=p(mt);La=o(ai,"AutoTokenizer.train_new_from_iterator()"),ai.forEach(t),Sa=o(da,". To see this in action, let\u2019s say we want to train GPT-2 from scratch, but in a language other than English. Our first task will be to gather lots of data in that language in a training corpus. To provide examples everyone will be able to understand, we won\u2019t use a language like Russian or Chinese here, but rather a specialized English language: Python code."),da.forEach(t),rs=c(e),v=l(e,"P",{});var I=p(v);Ia=o(I,"The "),ne=l(I,"A",{href:!0,rel:!0});var ni=p(ne);Na=o(ni,"\u{1F917} Datasets"),ni.forEach(t),Ga=o(I," library can help us assemble a corpus of Python source code. We\u2019ll use the usual "),gt=l(I,"CODE",{});var oi=p(gt);Ma=o(oi,"load_dataset()"),oi.forEach(t),Ra=o(I," function to download and cache the "),oe=l(I,"A",{href:!0,rel:!0});var ii=p(oe);Ua=o(ii,"CodeSearchNet"),ii.forEach(t),Ha=o(I," dataset. This dataset was created for the "),ie=l(I,"A",{href:!0,rel:!0});var ri=p(ie);Wa=o(ri,"CodeSearchNet challenge"),ri.forEach(t),Ya=o(I," and contains millions of functions from open source libraries on GitHub in several programming languages. Here, we will load the Python part of this dataset:"),I.forEach(t),ls=c(e),u(re.$$.fragment,e),ps=c(e),Ge=l(e,"P",{});var li=p(Ge);Ba=o(li,"We can have a look at the training split to see which columns we have access to:"),li.forEach(t),hs=c(e),u(le.$$.fragment,e),cs=c(e),u(pe.$$.fragment,e),fs=c(e),P=l(e,"P",{});var st=p(P);Fa=o(st,"We can see the dataset separates docstrings from code and suggests a tokenization of both. Here. we\u2019ll just use the "),wt=l(st,"CODE",{});var pi=p(wt);Ka=o(pi,"whole_func_string"),pi.forEach(t),Xa=o(st," column to train our tokenizer. We can look at an example of one these functions by indexing into the "),xt=l(st,"CODE",{});var hi=p(xt);Ja=o(hi,"train"),hi.forEach(t),Qa=o(st," split:"),st.forEach(t),us=c(e),u(he.$$.fragment,e),ds=c(e),Me=l(e,"P",{});var ci=p(Me);Za=o(ci,"which should print the following:"),ci.forEach(t),ms=c(e),u(ce.$$.fragment,e),gs=c(e),Y=l(e,"P",{});var ma=p(Y);Va=o(ma,"The first thing we need to do is transform the dataset into an "),_t=l(ma,"EM",{});var fi=p(_t);en=o(fi,"iterator"),fi.forEach(t),tn=o(ma," of lists of texts \u2014 for instance, a list of list of texts. Using lists of texts will enable our tokenizer to go faster (training on batches of texts instead of processing individual texts one by one), and it should be an iterator if we want to avoid having everything in memory at once. If your corpus is huge, you will want to take advantage of the fact that \u{1F917} Datasets does not load everything into RAM but stores the elements of the dataset on disk."),ma.forEach(t),ws=c(e),Re=l(e,"P",{});var ui=p(Re);sn=o(ui,"Doing the following would create a list of lists of 1,000 texts each, but would load everything in memory:"),ui.forEach(t),xs=c(e),u(fe.$$.fragment,e),_s=c(e),Ue=l(e,"P",{});var di=p(Ue);an=o(di,"Using a Python generator, we can avoid Python loading anything into memory until it\u2019s actually necessary. To create such a generator, you just to need to replace the brackets with parentheses:"),di.forEach(t),ks=c(e),u(ue.$$.fragment,e),ys=c(e),A=l(e,"P",{});var at=p(A);nn=o(at,"This line of code doesn\u2019t fetch any elements of the dataset; it just creates an object you can use in a Python "),kt=l(at,"CODE",{});var mi=p(kt);on=o(mi,"for"),mi.forEach(t),rn=o(at," loop. The texts will only be loaded when you need them (that is, when you\u2019re at the step of the "),yt=l(at,"CODE",{});var gi=p(yt);ln=o(gi,"for"),gi.forEach(t),pn=o(at," loop that requires them), and only 1,000 texts at a time will be loaded. This way you won\u2019t exhaust all your memory even if you are processing a huge dataset."),at.forEach(t),bs=c(e),He=l(e,"P",{});var wi=p(He);hn=o(wi,"The problem with a generator object is that it can only be used once. So, instead of this giving us the list of the first 10 digits twice:"),wi.forEach(t),$s=c(e),u(de.$$.fragment,e),js=c(e),We=l(e,"P",{});var xi=p(We);cn=o(xi,"we get them once and then an empty list:"),xi.forEach(t),vs=c(e),u(me.$$.fragment,e),zs=c(e),Ye=l(e,"P",{});var _i=p(Ye);fn=o(_i,"That\u2019s why we define a function that returns a generator instead:"),_i.forEach(t),Es=c(e),u(ge.$$.fragment,e),Ts=c(e),C=l(e,"P",{});var nt=p(C);un=o(nt,"You can also define your generator inside a "),bt=l(nt,"CODE",{});var ki=p(bt);dn=o(ki,"for"),ki.forEach(t),mn=o(nt," loop by using the "),$t=l(nt,"CODE",{});var yi=p($t);gn=o(yi,"yield"),yi.forEach(t),wn=o(nt," statement:"),nt.forEach(t),qs=c(e),u(we.$$.fragment,e),Ps=c(e),Be=l(e,"P",{});var bi=p(Be);xn=o(bi,"which will produce the exact same generator as before, but allows you to use more complex logic than you can in a list comprehension."),bi.forEach(t),As=c(e),G=l(e,"H2",{class:!0});var ga=p(G);B=l(ga,"A",{id:!0,class:!0,href:!0});var $i=p(B);jt=l($i,"SPAN",{});var ji=p(jt);u(xe.$$.fragment,ji),ji.forEach(t),$i.forEach(t),_n=c(ga),vt=l(ga,"SPAN",{});var vi=p(vt);kn=o(vi,"Training a new tokenizer"),vi.forEach(t),ga.forEach(t),Cs=c(e),Fe=l(e,"P",{});var zi=p(Fe);yn=o(zi,"Now that we have our corpus in the form of an iterator of batches of texts, we are ready to train a new tokenizer. To do this, we first need to load the tokenizer we want to pair with our model (here, GPT-2):"),zi.forEach(t),Ds=c(e),u(_e.$$.fragment,e),Os=c(e),Ke=l(e,"P",{});var Ei=p(Ke);bn=o(Ei,"Even though we are going to train a new tokenizer, it\u2019s a good idea to do this to avoid starting entirely from scratch. This way, we won\u2019t have to specify anything about the tokenization algorithm or the special tokens we want to use; our new tokenizer will be exactly the same as GPT-2, and the only thing that will change is the vocabulary, which will be determined by the training on our corpus."),Ei.forEach(t),Ls=c(e),Xe=l(e,"P",{});var Ti=p(Xe);$n=o(Ti,"First let\u2019s have a look at how this tokenizer would treat an example function:"),Ti.forEach(t),Ss=c(e),u(ke.$$.fragment,e),Is=c(e),u(ye.$$.fragment,e),Ns=c(e),T=l(e,"P",{});var V=p(T);jn=o(V,"This tokenizer has a few special symbols, like "),zt=l(V,"CODE",{});var qi=p(zt);vn=o(qi,"\u0120"),qi.forEach(t),zn=o(V," and "),Et=l(V,"CODE",{});var Pi=p(Et);En=o(Pi,"\u010A"),Pi.forEach(t),Tn=o(V,", which denote spaces and newlines, respectively. As we can see, this is not too efficient: the tokenizer returns individual tokens for each space, when it could group together indentation levels (since having sets of four or eight spaces is going to be very common in code). It also split the function name a bit weirdly, not being used to seeing words with the "),Tt=l(V,"CODE",{});var Ai=p(Tt);qn=o(Ai,"_"),Ai.forEach(t),Pn=o(V," character."),V.forEach(t),Gs=c(e),F=l(e,"P",{});var wa=p(F);An=o(wa,"Let\u2019s train a new tokenizer and see if it solves those issues. For this, we\u2019ll use the method "),qt=l(wa,"CODE",{});var Ci=p(qt);Cn=o(Ci,"train_new_from_iterator()"),Ci.forEach(t),Dn=o(wa,":"),wa.forEach(t),Ms=c(e),u(be.$$.fragment,e),Rs=c(e),Je=l(e,"P",{});var Di=p(Je);On=o(Di,"This command might take a bit of time if your corpus is very large, but for this dataset of 1.6 GB of texts it\u2019s  blazing fast (1 minute 16 seconds on an AMD Ryzen 9 3900X CPU with 12 cores)."),Di.forEach(t),Us=c(e),D=l(e,"P",{});var ot=p(D);Ln=o(ot,"Note that "),Pt=l(ot,"CODE",{});var Oi=p(Pt);Sn=o(Oi,"AutoTokenizer.train_new_from_iterator()"),Oi.forEach(t),In=o(ot," only works if the tokenizer you are using is a \u201Cfast\u201D tokenizer. As you\u2019ll see in the next section, the \u{1F917} Transformers library contains two types of tokenizers: some are written purely in Python and others (the fast ones) are backed by the \u{1F917} Tokenizers library, which is written in the "),$e=l(ot,"A",{href:!0,rel:!0});var Li=p($e);Nn=o(Li,"Rust"),Li.forEach(t),Gn=o(ot," programming language. Python is the language most often used for data science and deep learning applications, but when anything needs to be parallelized to be fast, it has to be written in another language. For instance, the matrix multiplications that are at the core of the model computation are written in CUDA, an optimized C library for GPUs."),ot.forEach(t),Hs=c(e),K=l(e,"P",{});var xa=p(K);Mn=o(xa,"Training a brand new tokenizer in pure Python would be excruciatingly slow, which is why we developed the \u{1F917} Tokenizers library. Note that just as you didn\u2019t have to learn the CUDA language to be able to execute your model on a batch of inputs on a GPU, you won\u2019t need to learn Rust to use a fast tokenizer. The \u{1F917} Tokenizers library provides Python bindings for many methods that internally call some piece of code in Rust; for example, to parallelize the training of your new tokenizer or, as we saw in "),Qe=l(xa,"A",{href:!0});var Si=p(Qe);Rn=o(Si,"Chapter 3"),Si.forEach(t),Un=o(xa,", the tokenization of a batch of inputs."),xa.forEach(t),Ws=c(e),O=l(e,"P",{});var it=p(O);Hn=o(it,"Most of the Transformer models have a fast tokenizer available (there are some exceptions that you can check "),je=l(it,"A",{href:!0,rel:!0});var Ii=p(je);Wn=o(Ii,"here"),Ii.forEach(t),Yn=o(it,"), and the "),At=l(it,"CODE",{});var Ni=p(At);Bn=o(Ni,"AutoTokenizer"),Ni.forEach(t),Fn=o(it," API always selects the fast tokenizer for you if it\u2019s available. In the next section we\u2019ll take a look at some of the other special features fast tokenizers have, which will be really useful for tasks like token classification and question answering. Before diving into that, however, let\u2019s try our brand new tokenizer on the previous example:"),it.forEach(t),Ys=c(e),u(ve.$$.fragment,e),Bs=c(e),u(ze.$$.fragment,e),Fs=c(e),$=l(e,"P",{});var q=p($);Kn=o(q,"Here we again see the special symbols "),Ct=l(q,"CODE",{});var Gi=p(Ct);Xn=o(Gi,"\u0120"),Gi.forEach(t),Jn=o(q," and "),Dt=l(q,"CODE",{});var Mi=p(Dt);Qn=o(Mi,"\u010A"),Mi.forEach(t),Zn=o(q," that denote spaces and newlines, but we can also see that our tokenizer learned some tokens that are highly specific to a corpus of Python functions: for example, there is a "),Ot=l(q,"CODE",{});var Ri=p(Ot);Vn=o(Ri,"\u010A\u0120\u0120\u0120"),Ri.forEach(t),eo=o(q," token that represents an indentation, and a "),Lt=l(q,"CODE",{});var Ui=p(Lt);to=o(Ui,'\u0120"""'),Ui.forEach(t),so=o(q," token that represents the three quotes that start a docstring. The tokenizer also correctly split the function name on "),St=l(q,"CODE",{});var Hi=p(St);ao=o(Hi,"_"),Hi.forEach(t),no=o(q,". This is quite a compact representation; comparatively, using the plain English tokenizer on the same example will give us a longer sentence:"),q.forEach(t),Ks=c(e),u(Ee.$$.fragment,e),Xs=c(e),u(Te.$$.fragment,e),Js=c(e),Ze=l(e,"P",{});var Wi=p(Ze);oo=o(Wi,"Let\u2019s look at another example:"),Wi.forEach(t),Qs=c(e),u(qe.$$.fragment,e),Zs=c(e),u(Pe.$$.fragment,e),Vs=c(e),_=l(e,"P",{});var y=p(_);io=o(y,"In addition to the token corresponding to an indentation, here we can also see a token for a double indentation: "),It=l(y,"CODE",{});var Yi=p(It);ro=o(Yi,"\u010A\u0120\u0120\u0120\u0120\u0120\u0120\u0120"),Yi.forEach(t),lo=o(y,". The special Python words like "),Nt=l(y,"CODE",{});var Bi=p(Nt);po=o(Bi,"class"),Bi.forEach(t),ho=o(y,", "),Gt=l(y,"CODE",{});var Fi=p(Gt);co=o(Fi,"init"),Fi.forEach(t),fo=o(y,", "),Mt=l(y,"CODE",{});var Ki=p(Mt);uo=o(Ki,"call"),Ki.forEach(t),mo=o(y,", "),Rt=l(y,"CODE",{});var Xi=p(Rt);go=o(Xi,"self"),Xi.forEach(t),wo=o(y,", and "),Ut=l(y,"CODE",{});var Ji=p(Ut);xo=o(Ji,"return"),Ji.forEach(t),_o=o(y," are each tokenized as one token, and we can see that as well as splitting on "),Ht=l(y,"CODE",{});var Qi=p(Ht);ko=o(Qi,"_"),Qi.forEach(t),yo=o(y," and "),Wt=l(y,"CODE",{});var Zi=p(Wt);bo=o(Zi,"."),Zi.forEach(t),$o=o(y," the tokenizer correctly splits even camel-cased names: "),Yt=l(y,"CODE",{});var Vi=p(Yt);jo=o(Vi,"LinearLayer"),Vi.forEach(t),vo=o(y," is tokenized as "),Bt=l(y,"CODE",{});var er=p(Bt);zo=o(er,'["\u0120Linear", "Layer"]'),er.forEach(t),Eo=o(y,"."),y.forEach(t),ea=c(e),M=l(e,"H2",{class:!0});var _a=p(M);X=l(_a,"A",{id:!0,class:!0,href:!0});var tr=p(X);Ft=l(tr,"SPAN",{});var sr=p(Ft);u(Ae.$$.fragment,sr),sr.forEach(t),tr.forEach(t),To=c(_a),Kt=l(_a,"SPAN",{});var ar=p(Kt);qo=o(ar,"Saving the tokenizer"),ar.forEach(t),_a.forEach(t),ta=c(e),J=l(e,"P",{});var ka=p(J);Po=o(ka,"To make sure we can use it later, we need to save our new tokenizer. Like for models, this is done with the "),Xt=l(ka,"CODE",{});var nr=p(Xt);Ao=o(nr,"save_pretrained()"),nr.forEach(t),Co=o(ka," method:"),ka.forEach(t),sa=c(e),u(Ce.$$.fragment,e),aa=c(e),Q=l(e,"P",{});var ya=p(Q);Do=o(ya,"This will create a new folder named "),Jt=l(ya,"EM",{});var or=p(Jt);Oo=o(or,"code-search-net-tokenizer"),or.forEach(t),Lo=o(ya,", which will contain all the files the tokenizer needs to be reloaded. If you want to share this tokenizer with your colleagues and friends, you can upload it to the Hub by logging into your account. If you\u2019re working in a notebook, there\u2019s a convenience function to help you with this:"),ya.forEach(t),na=c(e),u(De.$$.fragment,e),oa=c(e),Ve=l(e,"P",{});var ir=p(Ve);So=o(ir,"This will display a widget where you can enter your Hugging Face login credentials. If you aren\u2019t working in a notebook, just type the following line in your terminal:"),ir.forEach(t),ia=c(e),u(Oe.$$.fragment,e),ra=c(e),et=l(e,"P",{});var rr=p(et);Io=o(rr,"Once you\u2019ve logged in, you can push your tokenizer by executing the following command:"),rr.forEach(t),la=c(e),u(Le.$$.fragment,e),pa=c(e),L=l(e,"P",{});var rt=p(L);No=o(rt,"This will create a new repository in your namespace with the name "),Qt=l(rt,"CODE",{});var lr=p(Qt);Go=o(lr,"code-search-net-tokenizer"),lr.forEach(t),Mo=o(rt,", containing the tokenizer file. You can then load the tokenizer from anywhere with the "),Zt=l(rt,"CODE",{});var pr=p(Zt);Ro=o(pr,"from_pretrained()"),pr.forEach(t),Uo=o(rt," method:"),rt.forEach(t),ha=c(e),u(Se.$$.fragment,e),ca=c(e),S=l(e,"P",{});var lt=p(S);Ho=o(lt,"You\u2019re now all set for training a language model from scratch and fine-tuning it on your task at hand! We\u2019ll get to that in "),tt=l(lt,"A",{href:!0});var hr=p(tt);Wo=o(hr,"Chapter 7"),hr.forEach(t),Yo=o(lt,", but first, in the rest of this chapter we\u2019ll take a closer look at fast tokenizers and explore in detail what actually happens when we call the method "),Vt=l(lt,"CODE",{});var cr=p(Vt);Bo=o(cr,"train_new_from_iterator()"),cr.forEach(t),Fo=o(lt,"."),lt.forEach(t),this.h()},h(){x(b,"name","hf:doc:metadata"),x(b,"content",JSON.stringify(br)),x(z,"id","training-a-new-tokenizer-from-an-old-one"),x(z,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),x(z,"href","#training-a-new-tokenizer-from-an-old-one"),x(j,"class","relative group"),x(Ne,"href","/course/chapter2"),x(H,"id","assembling-a-corpus"),x(H,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),x(H,"href","#assembling-a-corpus"),x(N,"class","relative group"),x(ne,"href","https://github.com/huggingface/datasets"),x(ne,"rel","nofollow"),x(oe,"href","https://huggingface.co/datasets/code_search_net"),x(oe,"rel","nofollow"),x(ie,"href","https://wandb.ai/github/CodeSearchNet/benchmark"),x(ie,"rel","nofollow"),x(B,"id","training-a-new-tokenizer"),x(B,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),x(B,"href","#training-a-new-tokenizer"),x(G,"class","relative group"),x($e,"href","https://www.rust-lang.org"),x($e,"rel","nofollow"),x(Qe,"href","/course/chapter3"),x(je,"href","https://huggingface.co/transformers/#supported-frameworks"),x(je,"rel","nofollow"),x(X,"id","saving-the-tokenizer"),x(X,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),x(X,"href","#saving-the-tokenizer"),x(M,"class","relative group"),x(tt,"href","/course/chapter7")},m(e,a){s(document.head,b),i(e,R,a),i(e,j,a),s(j,z),s(z,pt),d(ee,pt,null),s(j,$a),s(j,ht),s(ht,ja),i(e,ts,a),d(te,e,a),i(e,ss,a),i(e,E,a),s(E,va),s(E,Ne),s(Ne,za),s(E,Ea),s(E,ct),s(ct,Ta),s(E,qa),s(E,ft),s(ft,Pa),s(E,Aa),i(e,as,a),d(se,e,a),i(e,ns,a),d(U,e,a),i(e,os,a),i(e,N,a),s(N,H),s(H,ut),d(ae,ut,null),s(N,Ca),s(N,dt),s(dt,Da),i(e,is,a),i(e,W,a),s(W,Oa),s(W,mt),s(mt,La),s(W,Sa),i(e,rs,a),i(e,v,a),s(v,Ia),s(v,ne),s(ne,Na),s(v,Ga),s(v,gt),s(gt,Ma),s(v,Ra),s(v,oe),s(oe,Ua),s(v,Ha),s(v,ie),s(ie,Wa),s(v,Ya),i(e,ls,a),d(re,e,a),i(e,ps,a),i(e,Ge,a),s(Ge,Ba),i(e,hs,a),d(le,e,a),i(e,cs,a),d(pe,e,a),i(e,fs,a),i(e,P,a),s(P,Fa),s(P,wt),s(wt,Ka),s(P,Xa),s(P,xt),s(xt,Ja),s(P,Qa),i(e,us,a),d(he,e,a),i(e,ds,a),i(e,Me,a),s(Me,Za),i(e,ms,a),d(ce,e,a),i(e,gs,a),i(e,Y,a),s(Y,Va),s(Y,_t),s(_t,en),s(Y,tn),i(e,ws,a),i(e,Re,a),s(Re,sn),i(e,xs,a),d(fe,e,a),i(e,_s,a),i(e,Ue,a),s(Ue,an),i(e,ks,a),d(ue,e,a),i(e,ys,a),i(e,A,a),s(A,nn),s(A,kt),s(kt,on),s(A,rn),s(A,yt),s(yt,ln),s(A,pn),i(e,bs,a),i(e,He,a),s(He,hn),i(e,$s,a),d(de,e,a),i(e,js,a),i(e,We,a),s(We,cn),i(e,vs,a),d(me,e,a),i(e,zs,a),i(e,Ye,a),s(Ye,fn),i(e,Es,a),d(ge,e,a),i(e,Ts,a),i(e,C,a),s(C,un),s(C,bt),s(bt,dn),s(C,mn),s(C,$t),s($t,gn),s(C,wn),i(e,qs,a),d(we,e,a),i(e,Ps,a),i(e,Be,a),s(Be,xn),i(e,As,a),i(e,G,a),s(G,B),s(B,jt),d(xe,jt,null),s(G,_n),s(G,vt),s(vt,kn),i(e,Cs,a),i(e,Fe,a),s(Fe,yn),i(e,Ds,a),d(_e,e,a),i(e,Os,a),i(e,Ke,a),s(Ke,bn),i(e,Ls,a),i(e,Xe,a),s(Xe,$n),i(e,Ss,a),d(ke,e,a),i(e,Is,a),d(ye,e,a),i(e,Ns,a),i(e,T,a),s(T,jn),s(T,zt),s(zt,vn),s(T,zn),s(T,Et),s(Et,En),s(T,Tn),s(T,Tt),s(Tt,qn),s(T,Pn),i(e,Gs,a),i(e,F,a),s(F,An),s(F,qt),s(qt,Cn),s(F,Dn),i(e,Ms,a),d(be,e,a),i(e,Rs,a),i(e,Je,a),s(Je,On),i(e,Us,a),i(e,D,a),s(D,Ln),s(D,Pt),s(Pt,Sn),s(D,In),s(D,$e),s($e,Nn),s(D,Gn),i(e,Hs,a),i(e,K,a),s(K,Mn),s(K,Qe),s(Qe,Rn),s(K,Un),i(e,Ws,a),i(e,O,a),s(O,Hn),s(O,je),s(je,Wn),s(O,Yn),s(O,At),s(At,Bn),s(O,Fn),i(e,Ys,a),d(ve,e,a),i(e,Bs,a),d(ze,e,a),i(e,Fs,a),i(e,$,a),s($,Kn),s($,Ct),s(Ct,Xn),s($,Jn),s($,Dt),s(Dt,Qn),s($,Zn),s($,Ot),s(Ot,Vn),s($,eo),s($,Lt),s(Lt,to),s($,so),s($,St),s(St,ao),s($,no),i(e,Ks,a),d(Ee,e,a),i(e,Xs,a),d(Te,e,a),i(e,Js,a),i(e,Ze,a),s(Ze,oo),i(e,Qs,a),d(qe,e,a),i(e,Zs,a),d(Pe,e,a),i(e,Vs,a),i(e,_,a),s(_,io),s(_,It),s(It,ro),s(_,lo),s(_,Nt),s(Nt,po),s(_,ho),s(_,Gt),s(Gt,co),s(_,fo),s(_,Mt),s(Mt,uo),s(_,mo),s(_,Rt),s(Rt,go),s(_,wo),s(_,Ut),s(Ut,xo),s(_,_o),s(_,Ht),s(Ht,ko),s(_,yo),s(_,Wt),s(Wt,bo),s(_,$o),s(_,Yt),s(Yt,jo),s(_,vo),s(_,Bt),s(Bt,zo),s(_,Eo),i(e,ea,a),i(e,M,a),s(M,X),s(X,Ft),d(Ae,Ft,null),s(M,To),s(M,Kt),s(Kt,qo),i(e,ta,a),i(e,J,a),s(J,Po),s(J,Xt),s(Xt,Ao),s(J,Co),i(e,sa,a),d(Ce,e,a),i(e,aa,a),i(e,Q,a),s(Q,Do),s(Q,Jt),s(Jt,Oo),s(Q,Lo),i(e,na,a),d(De,e,a),i(e,oa,a),i(e,Ve,a),s(Ve,So),i(e,ia,a),d(Oe,e,a),i(e,ra,a),i(e,et,a),s(et,Io),i(e,la,a),d(Le,e,a),i(e,pa,a),i(e,L,a),s(L,No),s(L,Qt),s(Qt,Go),s(L,Mo),s(L,Zt),s(Zt,Ro),s(L,Uo),i(e,ha,a),d(Se,e,a),i(e,ca,a),i(e,S,a),s(S,Ho),s(S,tt),s(tt,Wo),s(S,Yo),s(S,Vt),s(Vt,Bo),s(S,Fo),fa=!0},p(e,[a]){const Ie={};a&2&&(Ie.$$scope={dirty:a,ctx:e}),U.$set(Ie)},i(e){fa||(m(ee.$$.fragment,e),m(te.$$.fragment,e),m(se.$$.fragment,e),m(U.$$.fragment,e),m(ae.$$.fragment,e),m(re.$$.fragment,e),m(le.$$.fragment,e),m(pe.$$.fragment,e),m(he.$$.fragment,e),m(ce.$$.fragment,e),m(fe.$$.fragment,e),m(ue.$$.fragment,e),m(de.$$.fragment,e),m(me.$$.fragment,e),m(ge.$$.fragment,e),m(we.$$.fragment,e),m(xe.$$.fragment,e),m(_e.$$.fragment,e),m(ke.$$.fragment,e),m(ye.$$.fragment,e),m(be.$$.fragment,e),m(ve.$$.fragment,e),m(ze.$$.fragment,e),m(Ee.$$.fragment,e),m(Te.$$.fragment,e),m(qe.$$.fragment,e),m(Pe.$$.fragment,e),m(Ae.$$.fragment,e),m(Ce.$$.fragment,e),m(De.$$.fragment,e),m(Oe.$$.fragment,e),m(Le.$$.fragment,e),m(Se.$$.fragment,e),fa=!0)},o(e){g(ee.$$.fragment,e),g(te.$$.fragment,e),g(se.$$.fragment,e),g(U.$$.fragment,e),g(ae.$$.fragment,e),g(re.$$.fragment,e),g(le.$$.fragment,e),g(pe.$$.fragment,e),g(he.$$.fragment,e),g(ce.$$.fragment,e),g(fe.$$.fragment,e),g(ue.$$.fragment,e),g(de.$$.fragment,e),g(me.$$.fragment,e),g(ge.$$.fragment,e),g(we.$$.fragment,e),g(xe.$$.fragment,e),g(_e.$$.fragment,e),g(ke.$$.fragment,e),g(ye.$$.fragment,e),g(be.$$.fragment,e),g(ve.$$.fragment,e),g(ze.$$.fragment,e),g(Ee.$$.fragment,e),g(Te.$$.fragment,e),g(qe.$$.fragment,e),g(Pe.$$.fragment,e),g(Ae.$$.fragment,e),g(Ce.$$.fragment,e),g(De.$$.fragment,e),g(Oe.$$.fragment,e),g(Le.$$.fragment,e),g(Se.$$.fragment,e),fa=!1},d(e){t(b),e&&t(R),e&&t(j),w(ee),e&&t(ts),w(te,e),e&&t(ss),e&&t(E),e&&t(as),w(se,e),e&&t(ns),w(U,e),e&&t(os),e&&t(N),w(ae),e&&t(is),e&&t(W),e&&t(rs),e&&t(v),e&&t(ls),w(re,e),e&&t(ps),e&&t(Ge),e&&t(hs),w(le,e),e&&t(cs),w(pe,e),e&&t(fs),e&&t(P),e&&t(us),w(he,e),e&&t(ds),e&&t(Me),e&&t(ms),w(ce,e),e&&t(gs),e&&t(Y),e&&t(ws),e&&t(Re),e&&t(xs),w(fe,e),e&&t(_s),e&&t(Ue),e&&t(ks),w(ue,e),e&&t(ys),e&&t(A),e&&t(bs),e&&t(He),e&&t($s),w(de,e),e&&t(js),e&&t(We),e&&t(vs),w(me,e),e&&t(zs),e&&t(Ye),e&&t(Es),w(ge,e),e&&t(Ts),e&&t(C),e&&t(qs),w(we,e),e&&t(Ps),e&&t(Be),e&&t(As),e&&t(G),w(xe),e&&t(Cs),e&&t(Fe),e&&t(Ds),w(_e,e),e&&t(Os),e&&t(Ke),e&&t(Ls),e&&t(Xe),e&&t(Ss),w(ke,e),e&&t(Is),w(ye,e),e&&t(Ns),e&&t(T),e&&t(Gs),e&&t(F),e&&t(Ms),w(be,e),e&&t(Rs),e&&t(Je),e&&t(Us),e&&t(D),e&&t(Hs),e&&t(K),e&&t(Ws),e&&t(O),e&&t(Ys),w(ve,e),e&&t(Bs),w(ze,e),e&&t(Fs),e&&t($),e&&t(Ks),w(Ee,e),e&&t(Xs),w(Te,e),e&&t(Js),e&&t(Ze),e&&t(Qs),w(qe,e),e&&t(Zs),w(Pe,e),e&&t(Vs),e&&t(_),e&&t(ea),e&&t(M),w(Ae),e&&t(ta),e&&t(J),e&&t(sa),w(Ce,e),e&&t(aa),e&&t(Q),e&&t(na),w(De,e),e&&t(oa),e&&t(Ve),e&&t(ia),w(Oe,e),e&&t(ra),e&&t(et),e&&t(la),w(Le,e),e&&t(pa),e&&t(L),e&&t(ha),w(Se,e),e&&t(ca),e&&t(S)}}}const br={local:"training-a-new-tokenizer-from-an-old-one",sections:[{local:"assembling-a-corpus",title:"Assembling a corpus"},{local:"training-a-new-tokenizer",title:"Training a new tokenizer"},{local:"saving-the-tokenizer",title:"Saving the tokenizer"}],title:"Training a new tokenizer from an old one"};function $r(es){return gr(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Pr extends fr{constructor(b){super();ur(this,b,$r,yr,dr,{})}}export{Pr as default,br as metadata};
