import{S as du,i as cu,s as uu,e as i,t as a,c as r,a as h,h as o,d as s,g as p,G as t,k as u,w as $,m,x as j,y as x,q as _,o as w,B as E,b as T,M as mu,N as Zi,p as ss,v as fu,n as as}from"../../chunks/vendor-hf-doc-builder.js";import{T as Bs}from"../../chunks/Tip-hf-doc-builder.js";import{Y as er}from"../../chunks/Youtube-hf-doc-builder.js";import{I as Hs}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as A}from"../../chunks/CodeBlock-hf-doc-builder.js";import{C as hu}from"../../chunks/CourseFloatingBanner-hf-doc-builder.js";import{F as _u}from"../../chunks/FrameworkSwitchCourse-hf-doc-builder.js";function gu(K){let l,f;return l=new hu({props:{chapter:7,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter7/section3_tf.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter7/section3_tf.ipynb"}]}}),{c(){$(l.$$.fragment)},l(d){j(l.$$.fragment,d)},m(d,b){x(l,d,b),f=!0},i(d){f||(_(l.$$.fragment,d),f=!0)},o(d){w(l.$$.fragment,d),f=!1},d(d){E(l,d)}}}function wu(K){let l,f;return l=new hu({props:{chapter:7,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter7/section3_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter7/section3_pt.ipynb"}]}}),{c(){$(l.$$.fragment)},l(d){j(l.$$.fragment,d)},m(d,b){x(l,d,b),f=!0},i(d){f||(_(l.$$.fragment,d),f=!0)},o(d){w(l.$$.fragment,d),f=!1},d(d){E(l,d)}}}function ku(K){let l,f,d,b,z;return{c(){l=i("p"),f=a("\u{1F64B} If the terms \u201Cmasked language modeling\u201D and \u201Cpretrained model\u201D sound unfamiliar to you, go check out "),d=i("a"),b=a("Chapter 1"),z=a(", where we explain all these core concepts, complete with videos!"),this.h()},l(k){l=r(k,"P",{});var y=h(l);f=o(y,"\u{1F64B} If the terms \u201Cmasked language modeling\u201D and \u201Cpretrained model\u201D sound unfamiliar to you, go check out "),d=r(y,"A",{href:!0});var q=h(d);b=o(q,"Chapter 1"),q.forEach(s),z=o(y,", where we explain all these core concepts, complete with videos!"),y.forEach(s),this.h()},h(){T(d,"href","/course/chapter1")},m(k,y){p(k,l,y),t(l,f),t(l,d),t(d,b),t(l,z)},d(k){k&&s(l)}}}function bu(K){let l,f,d,b,z,k,y,q,g,D,F,O,M,N,P,W,B,J;return y=new A({props:{code:`from transformers import TFAutoModelForMaskedLM

model_checkpoint = "distilbert-base-uncased"
model = TFAutoModelForMaskedLM.from_pretrained(model_checkpoint)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForMaskedLM

model_checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>
model = TFAutoModelForMaskedLM.from_pretrained(model_checkpoint)`}}),P=new A({props:{code:"model.summary()",highlighted:"model.summary()"}}),B=new A({props:{code:`Model: "tf_distil_bert_for_masked_lm"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
distilbert (TFDistilBertMain multiple                  66362880  
_________________________________________________________________
vocab_transform (Dense)      multiple                  590592    
_________________________________________________________________
vocab_layer_norm (LayerNorma multiple                  1536      
_________________________________________________________________
vocab_projector (TFDistilBer multiple                  23866170  
=================================================================
Total params: 66,985,530
Trainable params: 66,985,530
Non-trainable params: 0
_________________________________________________________________`,highlighted:`Model: <span class="hljs-string">&quot;tf_distil_bert_for_masked_lm&quot;</span>
_________________________________________________________________
Layer (<span class="hljs-built_in">type</span>)                 Output Shape              Param <span class="hljs-comment">#   </span>
=================================================================
distilbert (TFDistilBertMain multiple                  <span class="hljs-number">66362880</span>  
_________________________________________________________________
vocab_transform (Dense)      multiple                  <span class="hljs-number">590592</span>    
_________________________________________________________________
vocab_layer_norm (LayerNorma multiple                  <span class="hljs-number">1536</span>      
_________________________________________________________________
vocab_projector (TFDistilBer multiple                  <span class="hljs-number">23866170</span>  
=================================================================
Total params: <span class="hljs-number">66</span>,<span class="hljs-number">985</span>,<span class="hljs-number">530</span>
Trainable params: <span class="hljs-number">66</span>,<span class="hljs-number">985</span>,<span class="hljs-number">530</span>
Non-trainable params: <span class="hljs-number">0</span>
_________________________________________________________________`}}),{c(){l=i("p"),f=a("Let\u2019s go ahead and download DistilBERT using the "),d=i("code"),b=a("AutoModelForMaskedLM"),z=a(" class:"),k=u(),$(y.$$.fragment),q=u(),g=i("p"),D=a("We can see how many parameters this model has by calling the "),F=i("code"),O=a("summary()"),M=a(" method:"),N=u(),$(P.$$.fragment),W=u(),$(B.$$.fragment)},l(v){l=r(v,"P",{});var R=h(l);f=o(R,"Let\u2019s go ahead and download DistilBERT using the "),d=r(R,"CODE",{});var U=h(d);b=o(U,"AutoModelForMaskedLM"),U.forEach(s),z=o(R," class:"),R.forEach(s),k=m(v),j(y.$$.fragment,v),q=m(v),g=r(v,"P",{});var Y=h(g);D=o(Y,"We can see how many parameters this model has by calling the "),F=r(Y,"CODE",{});var X=h(F);O=o(X,"summary()"),X.forEach(s),M=o(Y," method:"),Y.forEach(s),N=m(v),j(P.$$.fragment,v),W=m(v),j(B.$$.fragment,v)},m(v,R){p(v,l,R),t(l,f),t(l,d),t(d,b),t(l,z),p(v,k,R),x(y,v,R),p(v,q,R),p(v,g,R),t(g,D),t(g,F),t(F,O),t(g,M),p(v,N,R),x(P,v,R),p(v,W,R),x(B,v,R),J=!0},i(v){J||(_(y.$$.fragment,v),_(P.$$.fragment,v),_(B.$$.fragment,v),J=!0)},o(v){w(y.$$.fragment,v),w(P.$$.fragment,v),w(B.$$.fragment,v),J=!1},d(v){v&&s(l),v&&s(k),E(y,v),v&&s(q),v&&s(g),v&&s(N),E(P,v),v&&s(W),E(B,v)}}}function yu(K){let l,f,d,b,z,k,y,q,g,D,F,O,M,N,P,W,B,J;return y=new A({props:{code:`from transformers import AutoModelForMaskedLM

model_checkpoint = "distilbert-base-uncased"
model = AutoModelForMaskedLM.from_pretrained(model_checkpoint)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForMaskedLM

model_checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>
model = AutoModelForMaskedLM.from_pretrained(model_checkpoint)`}}),P=new A({props:{code:`distilbert_num_parameters = model.num_parameters() / 1_000_000
print(f"'>>> DistilBERT number of parameters: {round(distilbert_num_parameters)}M'")
print(f"'>>> BERT number of parameters: 110M'")`,highlighted:`distilbert_num_parameters = model.num_parameters() / <span class="hljs-number">1_000_000</span>
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&#x27;&gt;&gt;&gt; DistilBERT number of parameters: <span class="hljs-subst">{<span class="hljs-built_in">round</span>(distilbert_num_parameters)}</span>M&#x27;&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&#x27;&gt;&gt;&gt; BERT number of parameters: 110M&#x27;&quot;</span>)`}}),B=new A({props:{code:`'>>> DistilBERT number of parameters: 67M'
'>>> BERT number of parameters: 110M'`,highlighted:`<span class="hljs-string">&#x27;&gt;&gt;&gt; DistilBERT number of parameters: 67M&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; BERT number of parameters: 110M&#x27;</span>`}}),{c(){l=i("p"),f=a("Let\u2019s go ahead and download DistilBERT using the "),d=i("code"),b=a("AutoModelForMaskedLM"),z=a(" class:"),k=u(),$(y.$$.fragment),q=u(),g=i("p"),D=a("We can see how many parameters this model has by calling the "),F=i("code"),O=a("num_parameters()"),M=a(" method:"),N=u(),$(P.$$.fragment),W=u(),$(B.$$.fragment)},l(v){l=r(v,"P",{});var R=h(l);f=o(R,"Let\u2019s go ahead and download DistilBERT using the "),d=r(R,"CODE",{});var U=h(d);b=o(U,"AutoModelForMaskedLM"),U.forEach(s),z=o(R," class:"),R.forEach(s),k=m(v),j(y.$$.fragment,v),q=m(v),g=r(v,"P",{});var Y=h(g);D=o(Y,"We can see how many parameters this model has by calling the "),F=r(Y,"CODE",{});var X=h(F);O=o(X,"num_parameters()"),X.forEach(s),M=o(Y," method:"),Y.forEach(s),N=m(v),j(P.$$.fragment,v),W=m(v),j(B.$$.fragment,v)},m(v,R){p(v,l,R),t(l,f),t(l,d),t(d,b),t(l,z),p(v,k,R),x(y,v,R),p(v,q,R),p(v,g,R),t(g,D),t(g,F),t(F,O),t(g,M),p(v,N,R),x(P,v,R),p(v,W,R),x(B,v,R),J=!0},i(v){J||(_(y.$$.fragment,v),_(P.$$.fragment,v),_(B.$$.fragment,v),J=!0)},o(v){w(y.$$.fragment,v),w(P.$$.fragment,v),w(B.$$.fragment,v),J=!1},d(v){v&&s(l),v&&s(k),E(y,v),v&&s(q),v&&s(g),v&&s(N),E(P,v),v&&s(W),E(B,v)}}}function vu(K){let l,f;return l=new A({props:{code:`import numpy as np
import tensorflow as tf

inputs = tokenizer(text, return_tensors="np")
token_logits = model(**inputs).logits
# Find the location of [MASK] and extract its logits
mask_token_index = np.argwhere(inputs["input_ids"] == tokenizer.mask_token_id)[0, 1]
mask_token_logits = token_logits[0, mask_token_index, :]
# Pick the [MASK] candidates with the highest logits
# We negate the array before argsort to get the largest, not the smallest, logits
top_5_tokens = np.argsort(-mask_token_logits)[:5].tolist()

for token in top_5_tokens:
    print(f">>> {text.replace(tokenizer.mask_token, tokenizer.decode([token]))}")`,highlighted:`<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

inputs = tokenizer(text, return_tensors=<span class="hljs-string">&quot;np&quot;</span>)
token_logits = model(**inputs).logits
<span class="hljs-comment"># Find the location of [MASK] and extract its logits</span>
mask_token_index = np.argwhere(inputs[<span class="hljs-string">&quot;input_ids&quot;</span>] == tokenizer.mask_token_id)[<span class="hljs-number">0</span>, <span class="hljs-number">1</span>]
mask_token_logits = token_logits[<span class="hljs-number">0</span>, mask_token_index, :]
<span class="hljs-comment"># Pick the [MASK] candidates with the highest logits</span>
<span class="hljs-comment"># We negate the array before argsort to get the largest, not the smallest, logits</span>
top_5_tokens = np.argsort(-mask_token_logits)[:<span class="hljs-number">5</span>].tolist()

<span class="hljs-keyword">for</span> token <span class="hljs-keyword">in</span> top_5_tokens:
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&gt;&gt;&gt; <span class="hljs-subst">{text.replace(tokenizer.mask_token, tokenizer.decode([token]))}</span>&quot;</span>)`}}),{c(){$(l.$$.fragment)},l(d){j(l.$$.fragment,d)},m(d,b){x(l,d,b),f=!0},i(d){f||(_(l.$$.fragment,d),f=!0)},o(d){w(l.$$.fragment,d),f=!1},d(d){E(l,d)}}}function $u(K){let l,f;return l=new A({props:{code:`import torch

inputs = tokenizer(text, return_tensors="pt")
token_logits = model(**inputs).logits
# Find the location of [MASK] and extract its logits
mask_token_index = torch.where(inputs["input_ids"] == tokenizer.mask_token_id)[1]
mask_token_logits = token_logits[0, mask_token_index, :]
# Pick the [MASK] candidates with the highest logits
top_5_tokens = torch.topk(mask_token_logits, 5, dim=1).indices[0].tolist()

for token in top_5_tokens:
    print(f"'>>> {text.replace(tokenizer.mask_token, tokenizer.decode([token]))}'")`,highlighted:`<span class="hljs-keyword">import</span> torch

inputs = tokenizer(text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
token_logits = model(**inputs).logits
<span class="hljs-comment"># Find the location of [MASK] and extract its logits</span>
mask_token_index = torch.where(inputs[<span class="hljs-string">&quot;input_ids&quot;</span>] == tokenizer.mask_token_id)[<span class="hljs-number">1</span>]
mask_token_logits = token_logits[<span class="hljs-number">0</span>, mask_token_index, :]
<span class="hljs-comment"># Pick the [MASK] candidates with the highest logits</span>
top_5_tokens = torch.topk(mask_token_logits, <span class="hljs-number">5</span>, dim=<span class="hljs-number">1</span>).indices[<span class="hljs-number">0</span>].tolist()

<span class="hljs-keyword">for</span> token <span class="hljs-keyword">in</span> top_5_tokens:
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&#x27;&gt;&gt;&gt; <span class="hljs-subst">{text.replace(tokenizer.mask_token, tokenizer.decode([token]))}</span>&#x27;&quot;</span>)`}}),{c(){$(l.$$.fragment)},l(d){j(l.$$.fragment,d)},m(d,b){x(l,d,b),f=!0},i(d){f||(_(l.$$.fragment,d),f=!0)},o(d){w(l.$$.fragment,d),f=!1},d(d){E(l,d)}}}function ju(K){let l,f,d,b,z,k,y,q,g,D,F,O,M,N,P,W,B,J,v,R,U,Y,X,ue,Ae,ne;return{c(){l=i("p"),f=a("\u270F\uFE0F "),d=i("strong"),b=a("Try it out!"),z=a(" Create a random sample of the "),k=i("code"),y=a("unsupervised"),q=a(" split and verify that the labels are neither "),g=i("code"),D=a("0"),F=a(" nor "),O=i("code"),M=a("1"),N=a(". While you\u2019re at it, you could also check that the labels in the "),P=i("code"),W=a("train"),B=a(" and "),J=i("code"),v=a("test"),R=a(" splits are indeed "),U=i("code"),Y=a("0"),X=a(" or "),ue=i("code"),Ae=a("1"),ne=a(" \u2014 this is a useful sanity check that every NLP practitioner should perform at the start of a new project!")},l(le){l=r(le,"P",{});var V=h(l);f=o(V,"\u270F\uFE0F "),d=r(V,"STRONG",{});var _e=h(d);b=o(_e,"Try it out!"),_e.forEach(s),z=o(V," Create a random sample of the "),k=r(V,"CODE",{});var H=h(k);y=o(H,"unsupervised"),H.forEach(s),q=o(V," split and verify that the labels are neither "),g=r(V,"CODE",{});var ce=h(g);D=o(ce,"0"),ce.forEach(s),F=o(V," nor "),O=r(V,"CODE",{});var te=h(O);M=o(te,"1"),te.forEach(s),N=o(V,". While you\u2019re at it, you could also check that the labels in the "),P=r(V,"CODE",{});var ye=h(P);W=o(ye,"train"),ye.forEach(s),B=o(V," and "),J=r(V,"CODE",{});var qe=h(J);v=o(qe,"test"),qe.forEach(s),R=o(V," splits are indeed "),U=r(V,"CODE",{});var Q=h(U);Y=o(Q,"0"),Q.forEach(s),X=o(V," or "),ue=r(V,"CODE",{});var De=h(ue);Ae=o(De,"1"),De.forEach(s),ne=o(V," \u2014 this is a useful sanity check that every NLP practitioner should perform at the start of a new project!"),V.forEach(s)},m(le,V){p(le,l,V),t(l,f),t(l,d),t(d,b),t(l,z),t(l,k),t(k,y),t(l,q),t(l,g),t(g,D),t(l,F),t(l,O),t(O,M),t(l,N),t(l,P),t(P,W),t(l,B),t(l,J),t(J,v),t(l,R),t(l,U),t(U,Y),t(l,X),t(l,ue),t(ue,Ae),t(l,ne)},d(le){le&&s(l)}}}function xu(K){let l,f,d,b,z,k,y,q,g,D,F,O,M,N;return{c(){l=i("p"),f=a("\u270F\uFE0F "),d=i("strong"),b=a("Try it out!"),z=a(" Some Transformer models, like "),k=i("a"),y=a("BigBird"),q=a(" and "),g=i("a"),D=a("Longformer"),F=a(", have a much longer context length than BERT and other early Transformer models. Instantiate the tokenizer for one of these checkpoints and verify that the "),O=i("code"),M=a("model_max_length"),N=a(" agrees with what\u2019s quoted on its model card."),this.h()},l(P){l=r(P,"P",{});var W=h(l);f=o(W,"\u270F\uFE0F "),d=r(W,"STRONG",{});var B=h(d);b=o(B,"Try it out!"),B.forEach(s),z=o(W," Some Transformer models, like "),k=r(W,"A",{href:!0,rel:!0});var J=h(k);y=o(J,"BigBird"),J.forEach(s),q=o(W," and "),g=r(W,"A",{href:!0});var v=h(g);D=o(v,"Longformer"),v.forEach(s),F=o(W,", have a much longer context length than BERT and other early Transformer models. Instantiate the tokenizer for one of these checkpoints and verify that the "),O=r(W,"CODE",{});var R=h(O);M=o(R,"model_max_length"),R.forEach(s),N=o(W," agrees with what\u2019s quoted on its model card."),W.forEach(s),this.h()},h(){T(k,"href","https://huggingface.co/google/bigbird-roberta-base"),T(k,"rel","nofollow"),T(g,"href","hf.co/allenai/longformer-base-4096")},m(P,W){p(P,l,W),t(l,f),t(l,d),t(d,b),t(l,z),t(l,k),t(k,y),t(l,q),t(l,g),t(g,D),t(l,F),t(l,O),t(O,M),t(l,N)},d(P){P&&s(l)}}}function Eu(K){let l,f;return{c(){l=i("p"),f=a("Note that using a small chunk size can be detrimental in real-world scenarios, so you should use a size that corresponds to the use case you will apply your model to.")},l(d){l=r(d,"P",{});var b=h(l);f=o(b,"Note that using a small chunk size can be detrimental in real-world scenarios, so you should use a size that corresponds to the use case you will apply your model to."),b.forEach(s)},m(d,b){p(d,l,b),t(l,f)},d(d){d&&s(l)}}}function Tu(K){let l,f,d,b,z,k,y,q,g,D,F;return{c(){l=i("p"),f=a("\u270F\uFE0F "),d=i("strong"),b=a("Try it out!"),z=a(" Run the code snippet above several times to see the random masking happen in front of your very eyes! Also replace the "),k=i("code"),y=a("tokenizer.decode()"),q=a(" method with "),g=i("code"),D=a("tokenizer.convert_ids_to_tokens()"),F=a(" to see that sometimes a single token from a given word is masked, and not the others.")},l(O){l=r(O,"P",{});var M=h(l);f=o(M,"\u270F\uFE0F "),d=r(M,"STRONG",{});var N=h(d);b=o(N,"Try it out!"),N.forEach(s),z=o(M," Run the code snippet above several times to see the random masking happen in front of your very eyes! Also replace the "),k=r(M,"CODE",{});var P=h(k);y=o(P,"tokenizer.decode()"),P.forEach(s),q=o(M," method with "),g=r(M,"CODE",{});var W=h(g);D=o(W,"tokenizer.convert_ids_to_tokens()"),W.forEach(s),F=o(M," to see that sometimes a single token from a given word is masked, and not the others."),M.forEach(s)},m(O,M){p(O,l,M),t(l,f),t(l,d),t(d,b),t(l,z),t(l,k),t(k,y),t(l,q),t(l,g),t(g,D),t(l,F)},d(O){O&&s(l)}}}function iu(K){let l,f,d,b,z;return{c(){l=i("p"),f=a("One side effect of random masking is that our evaluation metrics will not be deterministic when using the "),d=i("code"),b=a("Trainer"),z=a(", since we use the same data collator for the training and test sets. We\u2019ll see later, when we look at fine-tuning with \u{1F917} Accelerate, how we can use the flexibility of a custom evaluation loop to freeze the randomness.")},l(k){l=r(k,"P",{});var y=h(l);f=o(y,"One side effect of random masking is that our evaluation metrics will not be deterministic when using the "),d=r(y,"CODE",{});var q=h(d);b=o(q,"Trainer"),q.forEach(s),z=o(y,", since we use the same data collator for the training and test sets. We\u2019ll see later, when we look at fine-tuning with \u{1F917} Accelerate, how we can use the flexibility of a custom evaluation loop to freeze the randomness."),y.forEach(s)},m(k,y){p(k,l,y),t(l,f),t(l,d),t(d,b),t(l,z)},d(k){k&&s(l)}}}function zu(K){let l,f;return l=new A({props:{code:`import collections
import numpy as np

from transformers.data.data_collator import tf_default_data_collator

wwm_probability = 0.2


def whole_word_masking_data_collator(features):
    for feature in features:
        word_ids = feature.pop("word_ids")

        # Create a map between words and corresponding token indices
        mapping = collections.defaultdict(list)
        current_word_index = -1
        current_word = None
        for idx, word_id in enumerate(word_ids):
            if word_id is not None:
                if word_id != current_word:
                    current_word = word_id
                    current_word_index += 1
                mapping[current_word_index].append(idx)

        # Randomly mask words
        mask = np.random.binomial(1, wwm_probability, (len(mapping),))
        input_ids = feature["input_ids"]
        labels = feature["labels"]
        new_labels = [-100] * len(labels)
        for word_id in np.where(mask)[0]:
            word_id = word_id.item()
            for idx in mapping[word_id]:
                new_labels[idx] = labels[idx]
                input_ids[idx] = tokenizer.mask_token_id
        feature["labels"] = new_labels

    return tf_default_data_collator(features)`,highlighted:`<span class="hljs-keyword">import</span> collections
<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np

<span class="hljs-keyword">from</span> transformers.data.data_collator <span class="hljs-keyword">import</span> tf_default_data_collator

wwm_probability = <span class="hljs-number">0.2</span>


<span class="hljs-keyword">def</span> <span class="hljs-title function_">whole_word_masking_data_collator</span>(<span class="hljs-params">features</span>):
    <span class="hljs-keyword">for</span> feature <span class="hljs-keyword">in</span> features:
        word_ids = feature.pop(<span class="hljs-string">&quot;word_ids&quot;</span>)

        <span class="hljs-comment"># Create a map between words and corresponding token indices</span>
        mapping = collections.defaultdict(<span class="hljs-built_in">list</span>)
        current_word_index = -<span class="hljs-number">1</span>
        current_word = <span class="hljs-literal">None</span>
        <span class="hljs-keyword">for</span> idx, word_id <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(word_ids):
            <span class="hljs-keyword">if</span> word_id <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>:
                <span class="hljs-keyword">if</span> word_id != current_word:
                    current_word = word_id
                    current_word_index += <span class="hljs-number">1</span>
                mapping[current_word_index].append(idx)

        <span class="hljs-comment"># Randomly mask words</span>
        mask = np.random.binomial(<span class="hljs-number">1</span>, wwm_probability, (<span class="hljs-built_in">len</span>(mapping),))
        input_ids = feature[<span class="hljs-string">&quot;input_ids&quot;</span>]
        labels = feature[<span class="hljs-string">&quot;labels&quot;</span>]
        new_labels = [-<span class="hljs-number">100</span>] * <span class="hljs-built_in">len</span>(labels)
        <span class="hljs-keyword">for</span> word_id <span class="hljs-keyword">in</span> np.where(mask)[<span class="hljs-number">0</span>]:
            word_id = word_id.item()
            <span class="hljs-keyword">for</span> idx <span class="hljs-keyword">in</span> mapping[word_id]:
                new_labels[idx] = labels[idx]
                input_ids[idx] = tokenizer.mask_token_id
        feature[<span class="hljs-string">&quot;labels&quot;</span>] = new_labels

    <span class="hljs-keyword">return</span> tf_default_data_collator(features)`}}),{c(){$(l.$$.fragment)},l(d){j(l.$$.fragment,d)},m(d,b){x(l,d,b),f=!0},i(d){f||(_(l.$$.fragment,d),f=!0)},o(d){w(l.$$.fragment,d),f=!1},d(d){E(l,d)}}}function Au(K){let l,f;return l=new A({props:{code:`import collections
import numpy as np

from transformers import default_data_collator

wwm_probability = 0.2


def whole_word_masking_data_collator(features):
    for feature in features:
        word_ids = feature.pop("word_ids")

        # Create a map between words and corresponding token indices
        mapping = collections.defaultdict(list)
        current_word_index = -1
        current_word = None
        for idx, word_id in enumerate(word_ids):
            if word_id is not None:
                if word_id != current_word:
                    current_word = word_id
                    current_word_index += 1
                mapping[current_word_index].append(idx)

        # Randomly mask words
        mask = np.random.binomial(1, wwm_probability, (len(mapping),))
        input_ids = feature["input_ids"]
        labels = feature["labels"]
        new_labels = [-100] * len(labels)
        for word_id in np.where(mask)[0]:
            word_id = word_id.item()
            for idx in mapping[word_id]:
                new_labels[idx] = labels[idx]
                input_ids[idx] = tokenizer.mask_token_id
        feature["labels"] = new_labels

    return default_data_collator(features)`,highlighted:`<span class="hljs-keyword">import</span> collections
<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np

<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> default_data_collator

wwm_probability = <span class="hljs-number">0.2</span>


<span class="hljs-keyword">def</span> <span class="hljs-title function_">whole_word_masking_data_collator</span>(<span class="hljs-params">features</span>):
    <span class="hljs-keyword">for</span> feature <span class="hljs-keyword">in</span> features:
        word_ids = feature.pop(<span class="hljs-string">&quot;word_ids&quot;</span>)

        <span class="hljs-comment"># Create a map between words and corresponding token indices</span>
        mapping = collections.defaultdict(<span class="hljs-built_in">list</span>)
        current_word_index = -<span class="hljs-number">1</span>
        current_word = <span class="hljs-literal">None</span>
        <span class="hljs-keyword">for</span> idx, word_id <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(word_ids):
            <span class="hljs-keyword">if</span> word_id <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>:
                <span class="hljs-keyword">if</span> word_id != current_word:
                    current_word = word_id
                    current_word_index += <span class="hljs-number">1</span>
                mapping[current_word_index].append(idx)

        <span class="hljs-comment"># Randomly mask words</span>
        mask = np.random.binomial(<span class="hljs-number">1</span>, wwm_probability, (<span class="hljs-built_in">len</span>(mapping),))
        input_ids = feature[<span class="hljs-string">&quot;input_ids&quot;</span>]
        labels = feature[<span class="hljs-string">&quot;labels&quot;</span>]
        new_labels = [-<span class="hljs-number">100</span>] * <span class="hljs-built_in">len</span>(labels)
        <span class="hljs-keyword">for</span> word_id <span class="hljs-keyword">in</span> np.where(mask)[<span class="hljs-number">0</span>]:
            word_id = word_id.item()
            <span class="hljs-keyword">for</span> idx <span class="hljs-keyword">in</span> mapping[word_id]:
                new_labels[idx] = labels[idx]
                input_ids[idx] = tokenizer.mask_token_id
        feature[<span class="hljs-string">&quot;labels&quot;</span>] = new_labels

    <span class="hljs-keyword">return</span> default_data_collator(features)`}}),{c(){$(l.$$.fragment)},l(d){j(l.$$.fragment,d)},m(d,b){x(l,d,b),f=!0},i(d){f||(_(l.$$.fragment,d),f=!0)},o(d){w(l.$$.fragment,d),f=!1},d(d){E(l,d)}}}function qu(K){let l,f,d,b,z,k,y,q,g,D,F;return{c(){l=i("p"),f=a("\u270F\uFE0F "),d=i("strong"),b=a("Try it out!"),z=a(" Run the code snippet above several times to see the random masking happen in front of your very eyes! Also replace the "),k=i("code"),y=a("tokenizer.decode()"),q=a(" method with "),g=i("code"),D=a("tokenizer.convert_ids_to_tokens()"),F=a(" to see that the tokens from a given word are always masked together.")},l(O){l=r(O,"P",{});var M=h(l);f=o(M,"\u270F\uFE0F "),d=r(M,"STRONG",{});var N=h(d);b=o(N,"Try it out!"),N.forEach(s),z=o(M," Run the code snippet above several times to see the random masking happen in front of your very eyes! Also replace the "),k=r(M,"CODE",{});var P=h(k);y=o(P,"tokenizer.decode()"),P.forEach(s),q=o(M," method with "),g=r(M,"CODE",{});var W=h(g);D=o(W,"tokenizer.convert_ids_to_tokens()"),W.forEach(s),F=o(M," to see that the tokens from a given word are always masked together."),M.forEach(s)},m(O,M){p(O,l,M),t(l,f),t(l,d),t(d,b),t(l,z),t(l,k),t(k,y),t(l,q),t(l,g),t(g,D),t(l,F)},d(O){O&&s(l)}}}function Du(K){let l,f,d,b,z,k,y,q,g,D,F,O,M,N,P,W,B,J,v,R,U,Y,X,ue,Ae,ne,le,V,_e,H,ce,te,ye,qe,Q,De,Ke,ie,Be,ge,Me,Je,re,pe,Qe,je,Re,Xe,we,Z,xe,ee,He,se,me,ke,ve,Fe,$e,Ie,C,I,Ge,fe,rt,oe,at,Ne,Ee;return y=new A({props:{code:`from transformers import TrainingArguments

batch_size = 64
# Show the training loss with every epoch
logging_steps = len(downsampled_dataset["train"]) // batch_size
model_name = model_checkpoint.split("/")[-1]

training_args = TrainingArguments(
    output_dir=f"{model_name}-finetuned-imdb",
    overwrite_output_dir=True,
    evaluation_strategy="epoch",
    learning_rate=2e-5,
    weight_decay=0.01,
    per_device_train_batch_size=batch_size,
    per_device_eval_batch_size=batch_size,
    push_to_hub=True,
    fp16=True,
    logging_steps=logging_steps,
)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrainingArguments

batch_size = <span class="hljs-number">64</span>
<span class="hljs-comment"># Show the training loss with every epoch</span>
logging_steps = <span class="hljs-built_in">len</span>(downsampled_dataset[<span class="hljs-string">&quot;train&quot;</span>]) // batch_size
model_name = model_checkpoint.split(<span class="hljs-string">&quot;/&quot;</span>)[-<span class="hljs-number">1</span>]

training_args = TrainingArguments(
    output_dir=<span class="hljs-string">f&quot;<span class="hljs-subst">{model_name}</span>-finetuned-imdb&quot;</span>,
    overwrite_output_dir=<span class="hljs-literal">True</span>,
    evaluation_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
    learning_rate=<span class="hljs-number">2e-5</span>,
    weight_decay=<span class="hljs-number">0.01</span>,
    per_device_train_batch_size=batch_size,
    per_device_eval_batch_size=batch_size,
    push_to_hub=<span class="hljs-literal">True</span>,
    fp16=<span class="hljs-literal">True</span>,
    logging_steps=logging_steps,
)`}}),$e=new A({props:{code:`from transformers import Trainer

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=downsampled_dataset["train"],
    eval_dataset=downsampled_dataset["test"],
    data_collator=data_collator,
    tokenizer=tokenizer,
)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Trainer

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=downsampled_dataset[<span class="hljs-string">&quot;train&quot;</span>],
    eval_dataset=downsampled_dataset[<span class="hljs-string">&quot;test&quot;</span>],
    data_collator=data_collator,
    tokenizer=tokenizer,
)`}}),{c(){l=i("p"),f=a("Once we\u2019re logged in, we can specify the arguments for the "),d=i("code"),b=a("Trainer"),z=a(":"),k=u(),$(y.$$.fragment),q=u(),g=i("p"),D=a("Here we tweaked a few of the default options, including "),F=i("code"),O=a("logging_steps"),M=a(" to ensure we track the training loss with each epoch. We\u2019ve also used "),N=i("code"),P=a("fp16=True"),W=a(" to enable mixed-precision training, which gives us another boost in speed. By default, the "),B=i("code"),J=a("Trainer"),v=a(" will remove any columns that are not part of the model\u2019s "),R=i("code"),U=a("forward()"),Y=a(" method. This means that if you\u2019re using the whole word masking collator, you\u2019ll also need to set "),X=i("code"),ue=a("remove_unused_columns=False"),Ae=a(" to ensure we don\u2019t lose the "),ne=i("code"),le=a("word_ids"),V=a(" column during training."),_e=u(),H=i("p"),ce=a("Note that you can specify the name of the repository you want to push to with the "),te=i("code"),ye=a("hub_model_id"),qe=a(" argument (in particular, you will have to use this argument to push to an organization). For instance, when we pushed the model to the "),Q=i("a"),De=i("code"),Ke=a("huggingface-course"),ie=a(" organization"),Be=a(", we added "),ge=i("code"),Me=a('hub_model_id="huggingface-course/distilbert-finetuned-imdb"'),Je=a(" to "),re=i("code"),pe=a("TrainingArguments"),Qe=a(". By default, the repository used will be in your namespace and named after the output directory you set, so in our case it will be "),je=i("code"),Re=a('"lewtun/distilbert-finetuned-imdb"'),Xe=a("."),we=u(),Z=i("p"),xe=a("We now have all the ingredients to instantiate the "),ee=i("code"),He=a("Trainer"),se=a(". Here we just use the standard "),me=i("code"),ke=a("data_collator"),ve=a(", but you can try the whole word masking collator and compare the results as an exercise:"),Fe=u(),$($e.$$.fragment),Ie=u(),C=i("p"),I=a("We\u2019re now ready to run "),Ge=i("code"),fe=a("trainer.train()"),rt=a(" \u2014 but before doing so let\u2019s briefly look at "),oe=i("em"),at=a("perplexity"),Ne=a(", which is a common metric to evaluate the performance of language models."),this.h()},l(S){l=r(S,"P",{});var G=h(l);f=o(G,"Once we\u2019re logged in, we can specify the arguments for the "),d=r(G,"CODE",{});var he=h(d);b=o(he,"Trainer"),he.forEach(s),z=o(G,":"),G.forEach(s),k=m(S),j(y.$$.fragment,S),q=m(S),g=r(S,"P",{});var de=h(g);D=o(de,"Here we tweaked a few of the default options, including "),F=r(de,"CODE",{});var Pe=h(F);O=o(Pe,"logging_steps"),Pe.forEach(s),M=o(de," to ensure we track the training loss with each epoch. We\u2019ve also used "),N=r(de,"CODE",{});var Ue=h(N);P=o(Ue,"fp16=True"),Ue.forEach(s),W=o(de," to enable mixed-precision training, which gives us another boost in speed. By default, the "),B=r(de,"CODE",{});var pt=h(B);J=o(pt,"Trainer"),pt.forEach(s),v=o(de," will remove any columns that are not part of the model\u2019s "),R=r(de,"CODE",{});var Te=h(R);U=o(Te,"forward()"),Te.forEach(s),Y=o(de," method. This means that if you\u2019re using the whole word masking collator, you\u2019ll also need to set "),X=r(de,"CODE",{});var Ce=h(X);ue=o(Ce,"remove_unused_columns=False"),Ce.forEach(s),Ae=o(de," to ensure we don\u2019t lose the "),ne=r(de,"CODE",{});var ht=h(ne);le=o(ht,"word_ids"),ht.forEach(s),V=o(de," column during training."),de.forEach(s),_e=m(S),H=r(S,"P",{});var ae=h(H);ce=o(ae,"Note that you can specify the name of the repository you want to push to with the "),te=r(ae,"CODE",{});var ot=h(te);ye=o(ot,"hub_model_id"),ot.forEach(s),qe=o(ae," argument (in particular, you will have to use this argument to push to an organization). For instance, when we pushed the model to the "),Q=r(ae,"A",{href:!0,rel:!0});var Ye=h(Q);De=r(Ye,"CODE",{});var Et=h(De);Ke=o(Et,"huggingface-course"),Et.forEach(s),ie=o(Ye," organization"),Ye.forEach(s),Be=o(ae,", we added "),ge=r(ae,"CODE",{});var nt=h(ge);Me=o(nt,'hub_model_id="huggingface-course/distilbert-finetuned-imdb"'),nt.forEach(s),Je=o(ae," to "),re=r(ae,"CODE",{});var Tt=h(re);pe=o(Tt,"TrainingArguments"),Tt.forEach(s),Qe=o(ae,". By default, the repository used will be in your namespace and named after the output directory you set, so in our case it will be "),je=r(ae,"CODE",{});var ze=h(je);Re=o(ze,'"lewtun/distilbert-finetuned-imdb"'),ze.forEach(s),Xe=o(ae,"."),ae.forEach(s),we=m(S),Z=r(S,"P",{});var be=h(Z);xe=o(be,"We now have all the ingredients to instantiate the "),ee=r(be,"CODE",{});var _t=h(ee);He=o(_t,"Trainer"),_t.forEach(s),se=o(be,". Here we just use the standard "),me=r(be,"CODE",{});var dt=h(me);ke=o(dt,"data_collator"),dt.forEach(s),ve=o(be,", but you can try the whole word masking collator and compare the results as an exercise:"),be.forEach(s),Fe=m(S),j($e.$$.fragment,S),Ie=m(S),C=r(S,"P",{});var We=h(C);I=o(We,"We\u2019re now ready to run "),Ge=r(We,"CODE",{});var gt=h(Ge);fe=o(gt,"trainer.train()"),gt.forEach(s),rt=o(We," \u2014 but before doing so let\u2019s briefly look at "),oe=r(We,"EM",{});var Ve=h(oe);at=o(Ve,"perplexity"),Ve.forEach(s),Ne=o(We,", which is a common metric to evaluate the performance of language models."),We.forEach(s),this.h()},h(){T(Q,"href","https://huggingface.co/huggingface-course"),T(Q,"rel","nofollow")},m(S,G){p(S,l,G),t(l,f),t(l,d),t(d,b),t(l,z),p(S,k,G),x(y,S,G),p(S,q,G),p(S,g,G),t(g,D),t(g,F),t(F,O),t(g,M),t(g,N),t(N,P),t(g,W),t(g,B),t(B,J),t(g,v),t(g,R),t(R,U),t(g,Y),t(g,X),t(X,ue),t(g,Ae),t(g,ne),t(ne,le),t(g,V),p(S,_e,G),p(S,H,G),t(H,ce),t(H,te),t(te,ye),t(H,qe),t(H,Q),t(Q,De),t(De,Ke),t(Q,ie),t(H,Be),t(H,ge),t(ge,Me),t(H,Je),t(H,re),t(re,pe),t(H,Qe),t(H,je),t(je,Re),t(H,Xe),p(S,we,G),p(S,Z,G),t(Z,xe),t(Z,ee),t(ee,He),t(Z,se),t(Z,me),t(me,ke),t(Z,ve),p(S,Fe,G),x($e,S,G),p(S,Ie,G),p(S,C,G),t(C,I),t(C,Ge),t(Ge,fe),t(C,rt),t(C,oe),t(oe,at),t(C,Ne),Ee=!0},i(S){Ee||(_(y.$$.fragment,S),_($e.$$.fragment,S),Ee=!0)},o(S){w(y.$$.fragment,S),w($e.$$.fragment,S),Ee=!1},d(S){S&&s(l),S&&s(k),E(y,S),S&&s(q),S&&s(g),S&&s(_e),S&&s(H),S&&s(we),S&&s(Z),S&&s(Fe),E($e,S),S&&s(Ie),S&&s(C)}}}function Mu(K){let l,f,d,b,z,k,y,q,g,D,F,O,M,N,P,W,B,J,v,R,U,Y,X,ue,Ae,ne,le,V,_e,H,ce,te,ye,qe,Q,De,Ke,ie,Be,ge,Me,Je,re,pe,Qe,je,Re,Xe,we,Z,xe,ee,He,se,me,ke,ve,Fe,$e,Ie;return M=new A({props:{code:`tf_train_dataset = model.prepare_tf_dataset(
    downsampled_dataset["train"],
    collate_fn=data_collator,
    shuffle=True,
    batch_size=32,
)

tf_eval_dataset = model.prepare_tf_dataset(
    downsampled_dataset["test"],
    collate_fn=data_collator,
    shuffle=False,
    batch_size=32,
)`,highlighted:`tf_train_dataset = model.prepare_tf_dataset(
    downsampled_dataset[<span class="hljs-string">&quot;train&quot;</span>],
    collate_fn=data_collator,
    shuffle=<span class="hljs-literal">True</span>,
    batch_size=<span class="hljs-number">32</span>,
)

tf_eval_dataset = model.prepare_tf_dataset(
    downsampled_dataset[<span class="hljs-string">&quot;test&quot;</span>],
    collate_fn=data_collator,
    shuffle=<span class="hljs-literal">False</span>,
    batch_size=<span class="hljs-number">32</span>,
)`}}),Z=new A({props:{code:`from transformers import create_optimizer
from transformers.keras_callbacks import PushToHubCallback
import tensorflow as tf

num_train_steps = len(tf_train_dataset)
optimizer, schedule = create_optimizer(
    init_lr=2e-5,
    num_warmup_steps=1_000,
    num_train_steps=num_train_steps,
    weight_decay_rate=0.01,
)
model.compile(optimizer=optimizer)

# Train in mixed-precision float16
tf.keras.mixed_precision.set_global_policy("mixed_float16")

model_name = model_checkpoint.split("/")[-1]
callback = PushToHubCallback(
    output_dir=f"{model_name}-finetuned-imdb", tokenizer=tokenizer
)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> create_optimizer
<span class="hljs-keyword">from</span> transformers.keras_callbacks <span class="hljs-keyword">import</span> PushToHubCallback
<span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

num_train_steps = <span class="hljs-built_in">len</span>(tf_train_dataset)
optimizer, schedule = create_optimizer(
    init_lr=<span class="hljs-number">2e-5</span>,
    num_warmup_steps=<span class="hljs-number">1_000</span>,
    num_train_steps=num_train_steps,
    weight_decay_rate=<span class="hljs-number">0.01</span>,
)
model.<span class="hljs-built_in">compile</span>(optimizer=optimizer)

<span class="hljs-comment"># Train in mixed-precision float16</span>
tf.keras.mixed_precision.set_global_policy(<span class="hljs-string">&quot;mixed_float16&quot;</span>)

model_name = model_checkpoint.split(<span class="hljs-string">&quot;/&quot;</span>)[-<span class="hljs-number">1</span>]
callback = PushToHubCallback(
    output_dir=<span class="hljs-string">f&quot;<span class="hljs-subst">{model_name}</span>-finetuned-imdb&quot;</span>, tokenizer=tokenizer
)`}}),{c(){l=i("p"),f=a("Once we\u2019re logged in, we can create our "),d=i("code"),b=a("tf.data"),z=a(" datasets. To do so, we\u2019ll use the "),k=i("code"),y=a("prepare_tf_dataset()"),q=a(" method, which uses our model to automatically infer which columns should go into the dataset. If you want to control exactly which columns to use, you can use the "),g=i("code"),D=a("Dataset.to_tf_dataset()"),F=a(" method instead. To keep things simple, we\u2019ll just use the standard data collator here, but you can also try the whole word masking collator and compare the results as an exercise:"),O=u(),$(M.$$.fragment),N=u(),P=i("p"),W=a("Next, we set up our training hyperparameters and compile our model. We use the "),B=i("code"),J=a("create_optimizer()"),v=a(" function from the \u{1F917} Transformers library, which gives us an "),R=i("code"),U=a("AdamW"),Y=a(" optimizer with linear learning rate decay. We also use the model\u2019s built-in loss, which is the default when no loss is specified as an argument to "),X=i("code"),ue=a("compile()"),Ae=a(", and we set the training precision to "),ne=i("code"),le=a('"mixed_float16"'),V=a(". Note that if you\u2019re using a Colab GPU or other GPU that does not have accelerated float16 support, you should probably comment out that line."),_e=u(),H=i("p"),ce=a("In addition, we set up a "),te=i("code"),ye=a("PushToHubCallback"),qe=a(" that will save the model to the Hub after each epoch. You can specify the name of the repository you want to push to with the "),Q=i("code"),De=a("hub_model_id"),Ke=a(" argument (in particular, you will have to use this argument to push to an organization). For instance, to push the model to the "),ie=i("a"),Be=i("code"),ge=a("huggingface-course"),Me=a(" organization"),Je=a(", we added "),re=i("code"),pe=a('hub_model_id="huggingface-course/distilbert-finetuned-imdb"'),Qe=a(". By default, the repository used will be in your namespace and named after the output directory you set, so in our case it will be "),je=i("code"),Re=a('"lewtun/distilbert-finetuned-imdb"'),Xe=a("."),we=u(),$(Z.$$.fragment),xe=u(),ee=i("p"),He=a("We\u2019re now ready to run "),se=i("code"),me=a("model.fit()"),ke=a(" \u2014 but before doing so let\u2019s briefly look at "),ve=i("em"),Fe=a("perplexity"),$e=a(", which is a common metric to evaluate the performance of language models."),this.h()},l(C){l=r(C,"P",{});var I=h(l);f=o(I,"Once we\u2019re logged in, we can create our "),d=r(I,"CODE",{});var Ge=h(d);b=o(Ge,"tf.data"),Ge.forEach(s),z=o(I," datasets. To do so, we\u2019ll use the "),k=r(I,"CODE",{});var fe=h(k);y=o(fe,"prepare_tf_dataset()"),fe.forEach(s),q=o(I," method, which uses our model to automatically infer which columns should go into the dataset. If you want to control exactly which columns to use, you can use the "),g=r(I,"CODE",{});var rt=h(g);D=o(rt,"Dataset.to_tf_dataset()"),rt.forEach(s),F=o(I," method instead. To keep things simple, we\u2019ll just use the standard data collator here, but you can also try the whole word masking collator and compare the results as an exercise:"),I.forEach(s),O=m(C),j(M.$$.fragment,C),N=m(C),P=r(C,"P",{});var oe=h(P);W=o(oe,"Next, we set up our training hyperparameters and compile our model. We use the "),B=r(oe,"CODE",{});var at=h(B);J=o(at,"create_optimizer()"),at.forEach(s),v=o(oe," function from the \u{1F917} Transformers library, which gives us an "),R=r(oe,"CODE",{});var Ne=h(R);U=o(Ne,"AdamW"),Ne.forEach(s),Y=o(oe," optimizer with linear learning rate decay. We also use the model\u2019s built-in loss, which is the default when no loss is specified as an argument to "),X=r(oe,"CODE",{});var Ee=h(X);ue=o(Ee,"compile()"),Ee.forEach(s),Ae=o(oe,", and we set the training precision to "),ne=r(oe,"CODE",{});var S=h(ne);le=o(S,'"mixed_float16"'),S.forEach(s),V=o(oe,". Note that if you\u2019re using a Colab GPU or other GPU that does not have accelerated float16 support, you should probably comment out that line."),oe.forEach(s),_e=m(C),H=r(C,"P",{});var G=h(H);ce=o(G,"In addition, we set up a "),te=r(G,"CODE",{});var he=h(te);ye=o(he,"PushToHubCallback"),he.forEach(s),qe=o(G," that will save the model to the Hub after each epoch. You can specify the name of the repository you want to push to with the "),Q=r(G,"CODE",{});var de=h(Q);De=o(de,"hub_model_id"),de.forEach(s),Ke=o(G," argument (in particular, you will have to use this argument to push to an organization). For instance, to push the model to the "),ie=r(G,"A",{href:!0,rel:!0});var Pe=h(ie);Be=r(Pe,"CODE",{});var Ue=h(Be);ge=o(Ue,"huggingface-course"),Ue.forEach(s),Me=o(Pe," organization"),Pe.forEach(s),Je=o(G,", we added "),re=r(G,"CODE",{});var pt=h(re);pe=o(pt,'hub_model_id="huggingface-course/distilbert-finetuned-imdb"'),pt.forEach(s),Qe=o(G,". By default, the repository used will be in your namespace and named after the output directory you set, so in our case it will be "),je=r(G,"CODE",{});var Te=h(je);Re=o(Te,'"lewtun/distilbert-finetuned-imdb"'),Te.forEach(s),Xe=o(G,"."),G.forEach(s),we=m(C),j(Z.$$.fragment,C),xe=m(C),ee=r(C,"P",{});var Ce=h(ee);He=o(Ce,"We\u2019re now ready to run "),se=r(Ce,"CODE",{});var ht=h(se);me=o(ht,"model.fit()"),ht.forEach(s),ke=o(Ce," \u2014 but before doing so let\u2019s briefly look at "),ve=r(Ce,"EM",{});var ae=h(ve);Fe=o(ae,"perplexity"),ae.forEach(s),$e=o(Ce,", which is a common metric to evaluate the performance of language models."),Ce.forEach(s),this.h()},h(){T(ie,"href","https://huggingface.co/huggingface-course"),T(ie,"rel","nofollow")},m(C,I){p(C,l,I),t(l,f),t(l,d),t(d,b),t(l,z),t(l,k),t(k,y),t(l,q),t(l,g),t(g,D),t(l,F),p(C,O,I),x(M,C,I),p(C,N,I),p(C,P,I),t(P,W),t(P,B),t(B,J),t(P,v),t(P,R),t(R,U),t(P,Y),t(P,X),t(X,ue),t(P,Ae),t(P,ne),t(ne,le),t(P,V),p(C,_e,I),p(C,H,I),t(H,ce),t(H,te),t(te,ye),t(H,qe),t(H,Q),t(Q,De),t(H,Ke),t(H,ie),t(ie,Be),t(Be,ge),t(ie,Me),t(H,Je),t(H,re),t(re,pe),t(H,Qe),t(H,je),t(je,Re),t(H,Xe),p(C,we,I),x(Z,C,I),p(C,xe,I),p(C,ee,I),t(ee,He),t(ee,se),t(se,me),t(ee,ke),t(ee,ve),t(ve,Fe),t(ee,$e),Ie=!0},i(C){Ie||(_(M.$$.fragment,C),_(Z.$$.fragment,C),Ie=!0)},o(C){w(M.$$.fragment,C),w(Z.$$.fragment,C),Ie=!1},d(C){C&&s(l),C&&s(O),E(M,C),C&&s(N),C&&s(P),C&&s(_e),C&&s(H),C&&s(we),E(Z,C),C&&s(xe),C&&s(ee)}}}function Pu(K){let l,f,d,b,z,k,y,q;return y=new A({props:{code:`import math

eval_loss = model.evaluate(tf_eval_dataset)
print(f"Perplexity: {math.exp(eval_loss):.2f}")`,highlighted:`<span class="hljs-keyword">import</span> math

eval_loss = model.evaluate(tf_eval_dataset)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Perplexity: <span class="hljs-subst">{math.exp(eval_loss):<span class="hljs-number">.2</span>f}</span>&quot;</span>)`}}),{c(){l=i("p"),f=a("Assuming our test set consists mostly of sentences that are grammatically correct, then one way to measure the quality of our language model is to calculate the probabilities it assigns to the next word in all the sentences of the test set. High probabilities indicates that the model indicates that the model is not \u201Csurprised\u201D or \u201Cperplexed\u201D by the unseen examples, and suggests it has learned the basic patterns of grammar in the language. There are various mathematical definitions of perplexity, but the one we\u2019ll use defines it as the exponential of the cross-entropy loss. Thus, we can calculate the perplexity of our pretrained model by using the "),d=i("code"),b=a("model.evaluate()"),z=a(" method to compute the cross-entropy loss on the test set and then taking the exponential of the result:"),k=u(),$(y.$$.fragment)},l(g){l=r(g,"P",{});var D=h(l);f=o(D,"Assuming our test set consists mostly of sentences that are grammatically correct, then one way to measure the quality of our language model is to calculate the probabilities it assigns to the next word in all the sentences of the test set. High probabilities indicates that the model indicates that the model is not \u201Csurprised\u201D or \u201Cperplexed\u201D by the unseen examples, and suggests it has learned the basic patterns of grammar in the language. There are various mathematical definitions of perplexity, but the one we\u2019ll use defines it as the exponential of the cross-entropy loss. Thus, we can calculate the perplexity of our pretrained model by using the "),d=r(D,"CODE",{});var F=h(d);b=o(F,"model.evaluate()"),F.forEach(s),z=o(D," method to compute the cross-entropy loss on the test set and then taking the exponential of the result:"),D.forEach(s),k=m(g),j(y.$$.fragment,g)},m(g,D){p(g,l,D),t(l,f),t(l,d),t(d,b),t(l,z),p(g,k,D),x(y,g,D),q=!0},i(g){q||(_(y.$$.fragment,g),q=!0)},o(g){w(y.$$.fragment,g),q=!1},d(g){g&&s(l),g&&s(k),E(y,g)}}}function Cu(K){let l,f,d,b,z,k,y,q;return y=new A({props:{code:`import math

eval_results = trainer.evaluate()
print(f">>> Perplexity: {math.exp(eval_results['eval_loss']):.2f}")`,highlighted:`<span class="hljs-keyword">import</span> math

eval_results = trainer.evaluate()
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&gt;&gt;&gt; Perplexity: <span class="hljs-subst">{math.exp(eval_results[<span class="hljs-string">&#x27;eval_loss&#x27;</span>]):<span class="hljs-number">.2</span>f}</span>&quot;</span>)`}}),{c(){l=i("p"),f=a("Assuming our test set consists mostly of sentences that are grammatically correct, then one way to measure the quality of our language model is to calculate the probabilities it assigns to the next word in all the sentences of the test set. High probabilities indicates that the model is not \u201Csurprised\u201D or \u201Cperplexed\u201D by the unseen examples, and suggests it has learned the basic patterns of grammar in the language. There are various mathematical definitions of perplexity, but the one we\u2019ll use defines it as the exponential of the cross-entropy loss. Thus, we can calculate the perplexity of our pretrained model by using the "),d=i("code"),b=a("Trainer.evaluate()"),z=a(" function to compute the cross-entropy loss on the test set and then taking the exponential of the result:"),k=u(),$(y.$$.fragment)},l(g){l=r(g,"P",{});var D=h(l);f=o(D,"Assuming our test set consists mostly of sentences that are grammatically correct, then one way to measure the quality of our language model is to calculate the probabilities it assigns to the next word in all the sentences of the test set. High probabilities indicates that the model is not \u201Csurprised\u201D or \u201Cperplexed\u201D by the unseen examples, and suggests it has learned the basic patterns of grammar in the language. There are various mathematical definitions of perplexity, but the one we\u2019ll use defines it as the exponential of the cross-entropy loss. Thus, we can calculate the perplexity of our pretrained model by using the "),d=r(D,"CODE",{});var F=h(d);b=o(F,"Trainer.evaluate()"),F.forEach(s),z=o(D," function to compute the cross-entropy loss on the test set and then taking the exponential of the result:"),D.forEach(s),k=m(g),j(y.$$.fragment,g)},m(g,D){p(g,l,D),t(l,f),t(l,d),t(d,b),t(l,z),p(g,k,D),x(y,g,D),q=!0},i(g){q||(_(y.$$.fragment,g),q=!0)},o(g){w(y.$$.fragment,g),q=!1},d(g){g&&s(l),g&&s(k),E(y,g)}}}function Su(K){let l,f;return l=new A({props:{code:"model.fit(tf_train_dataset, validation_data=tf_eval_dataset, callbacks=[callback])",highlighted:"model.fit(tf_train_dataset, validation_data=tf_eval_dataset, callbacks=[callback])"}}),{c(){$(l.$$.fragment)},l(d){j(l.$$.fragment,d)},m(d,b){x(l,d,b),f=!0},i(d){f||(_(l.$$.fragment,d),f=!0)},o(d){w(l.$$.fragment,d),f=!1},d(d){E(l,d)}}}function Ou(K){let l,f;return l=new A({props:{code:"trainer.train()",highlighted:"trainer.train()"}}),{c(){$(l.$$.fragment)},l(d){j(l.$$.fragment,d)},m(d,b){x(l,d,b),f=!0},i(d){f||(_(l.$$.fragment,d),f=!0)},o(d){w(l.$$.fragment,d),f=!1},d(d){E(l,d)}}}function Lu(K){let l,f;return l=new A({props:{code:`eval_loss = model.evaluate(tf_eval_dataset)
print(f"Perplexity: {math.exp(eval_loss):.2f}")`,highlighted:`eval_loss = model.evaluate(tf_eval_dataset)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Perplexity: <span class="hljs-subst">{math.exp(eval_loss):<span class="hljs-number">.2</span>f}</span>&quot;</span>)`}}),{c(){$(l.$$.fragment)},l(d){j(l.$$.fragment,d)},m(d,b){x(l,d,b),f=!0},i(d){f||(_(l.$$.fragment,d),f=!0)},o(d){w(l.$$.fragment,d),f=!1},d(d){E(l,d)}}}function Ku(K){let l,f;return l=new A({props:{code:`eval_results = trainer.evaluate()
print(f">>> Perplexity: {math.exp(eval_results['eval_loss']):.2f}")`,highlighted:`eval_results = trainer.evaluate()
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&gt;&gt;&gt; Perplexity: <span class="hljs-subst">{math.exp(eval_results[<span class="hljs-string">&#x27;eval_loss&#x27;</span>]):<span class="hljs-number">.2</span>f}</span>&quot;</span>)`}}),{c(){$(l.$$.fragment)},l(d){j(l.$$.fragment,d)},m(d,b){x(l,d,b),f=!0},i(d){f||(_(l.$$.fragment,d),f=!0)},o(d){w(l.$$.fragment,d),f=!1},d(d){E(l,d)}}}function ru(K){let l,f,d,b,z;return b=new A({props:{code:"trainer.push_to_hub()",highlighted:"trainer.push_to_hub()"}}),{c(){l=i("p"),f=a("Once training is finished, we can push the model card with the training information to the Hub (the checkpoints are saved during training itself):"),d=u(),$(b.$$.fragment)},l(k){l=r(k,"P",{});var y=h(l);f=o(y,"Once training is finished, we can push the model card with the training information to the Hub (the checkpoints are saved during training itself):"),y.forEach(s),d=m(k),j(b.$$.fragment,k)},m(k,y){p(k,l,y),t(l,f),p(k,d,y),x(b,k,y),z=!0},i(k){z||(_(b.$$.fragment,k),z=!0)},o(k){w(b.$$.fragment,k),z=!1},d(k){k&&s(l),k&&s(d),E(b,k)}}}function Ru(K){let l,f,d,b,z;return{c(){l=i("p"),f=a("\u270F\uFE0F "),d=i("strong"),b=a("Your turn!"),z=a(" Run the training above after changing the data collator to the whole word masking collator. Do you get better results?")},l(k){l=r(k,"P",{});var y=h(l);f=o(y,"\u270F\uFE0F "),d=r(y,"STRONG",{});var q=h(d);b=o(q,"Your turn!"),q.forEach(s),z=o(y," Run the training above after changing the data collator to the whole word masking collator. Do you get better results?"),y.forEach(s)},m(k,y){p(k,l,y),t(l,f),t(l,d),t(d,b),t(l,z)},d(k){k&&s(l)}}}function pu(K){let l,f,d,b,z,k,y,q,g,D,F,O,M,N,P,W,B,J,v,R,U,Y,X,ue,Ae,ne,le,V,_e,H,ce,te,ye,qe,Q,De,Ke,ie,Be,ge,Me,Je,re,pe,Qe,je,Re,Xe,we,Z,xe,ee,He,se,me,ke,ve,Fe,$e,Ie,C,I,Ge,fe,rt,oe,at,Ne,Ee,S,G,he,de,Pe,Ue,pt,Te,Ce,ht,ae,ot,Ye,Et,nt,Tt,ze,be,_t,dt,We,gt,Ve,ds,Se,zt,Wt,At,cs,ct,Is,us;return y=new Hs({}),ye=new A({props:{code:`def insert_random_mask(batch):
    features = [dict(zip(batch, t)) for t in zip(*batch.values())]
    masked_inputs = data_collator(features)
    # Create a new "masked" column for each column in the dataset
    return {"masked_" + k: v.numpy() for k, v in masked_inputs.items()}`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">insert_random_mask</span>(<span class="hljs-params">batch</span>):
    features = [<span class="hljs-built_in">dict</span>(<span class="hljs-built_in">zip</span>(batch, t)) <span class="hljs-keyword">for</span> t <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(*batch.values())]
    masked_inputs = data_collator(features)
    <span class="hljs-comment"># Create a new &quot;masked&quot; column for each column in the dataset</span>
    <span class="hljs-keyword">return</span> {<span class="hljs-string">&quot;masked_&quot;</span> + k: v.numpy() <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> masked_inputs.items()}`}}),Me=new A({props:{code:`downsampled_dataset = downsampled_dataset.remove_columns(["word_ids"])
eval_dataset = downsampled_dataset["test"].map(
    insert_random_mask,
    batched=True,
    remove_columns=downsampled_dataset["test"].column_names,
)
eval_dataset = eval_dataset.rename_columns(
    {
        "masked_input_ids": "input_ids",
        "masked_attention_mask": "attention_mask",
        "masked_labels": "labels",
    }
)`,highlighted:`downsampled_dataset = downsampled_dataset.remove_columns([<span class="hljs-string">&quot;word_ids&quot;</span>])
eval_dataset = downsampled_dataset[<span class="hljs-string">&quot;test&quot;</span>].<span class="hljs-built_in">map</span>(
    insert_random_mask,
    batched=<span class="hljs-literal">True</span>,
    remove_columns=downsampled_dataset[<span class="hljs-string">&quot;test&quot;</span>].column_names,
)
eval_dataset = eval_dataset.rename_columns(
    {
        <span class="hljs-string">&quot;masked_input_ids&quot;</span>: <span class="hljs-string">&quot;input_ids&quot;</span>,
        <span class="hljs-string">&quot;masked_attention_mask&quot;</span>: <span class="hljs-string">&quot;attention_mask&quot;</span>,
        <span class="hljs-string">&quot;masked_labels&quot;</span>: <span class="hljs-string">&quot;labels&quot;</span>,
    }
)`}}),we=new A({props:{code:`from torch.utils.data import DataLoader
from transformers import default_data_collator

batch_size = 64
train_dataloader = DataLoader(
    downsampled_dataset["train"],
    shuffle=True,
    batch_size=batch_size,
    collate_fn=data_collator,
)
eval_dataloader = DataLoader(
    eval_dataset, batch_size=batch_size, collate_fn=default_data_collator
)`,highlighted:`<span class="hljs-keyword">from</span> torch.utils.data <span class="hljs-keyword">import</span> DataLoader
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> default_data_collator

batch_size = <span class="hljs-number">64</span>
train_dataloader = DataLoader(
    downsampled_dataset[<span class="hljs-string">&quot;train&quot;</span>],
    shuffle=<span class="hljs-literal">True</span>,
    batch_size=batch_size,
    collate_fn=data_collator,
)
eval_dataloader = DataLoader(
    eval_dataset, batch_size=batch_size, collate_fn=default_data_collator
)`}}),se=new A({props:{code:"model = AutoModelForMaskedLM.from_pretrained(model_checkpoint)",highlighted:'model = <span class="hljs-module-access"><span class="hljs-module"><span class="hljs-identifier">AutoModelForMaskedLM</span>.</span></span>from<span class="hljs-constructor">_pretrained(<span class="hljs-params">model_checkpoint</span>)</span>'}}),I=new A({props:{code:`from torch.optim import AdamW

optimizer = AdamW(model.parameters(), lr=5e-5)`,highlighted:`<span class="hljs-keyword">from</span> torch.optim <span class="hljs-keyword">import</span> AdamW

optimizer = AdamW(model.parameters(), lr=<span class="hljs-number">5e-5</span>)`}}),S=new A({props:{code:`from accelerate import Accelerator

accelerator = Accelerator()
model, optimizer, train_dataloader, eval_dataloader = accelerator.prepare(
    model, optimizer, train_dataloader, eval_dataloader
)`,highlighted:`<span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> Accelerator

accelerator = Accelerator()
model, optimizer, train_dataloader, eval_dataloader = accelerator.prepare(
    model, optimizer, train_dataloader, eval_dataloader
)`}}),Ue=new A({props:{code:`from transformers import get_scheduler

num_train_epochs = 3
num_update_steps_per_epoch = len(train_dataloader)
num_training_steps = num_train_epochs * num_update_steps_per_epoch

lr_scheduler = get_scheduler(
    "linear",
    optimizer=optimizer,
    num_warmup_steps=0,
    num_training_steps=num_training_steps,
)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> get_scheduler

num_train_epochs = <span class="hljs-number">3</span>
num_update_steps_per_epoch = <span class="hljs-built_in">len</span>(train_dataloader)
num_training_steps = num_train_epochs * num_update_steps_per_epoch

lr_scheduler = get_scheduler(
    <span class="hljs-string">&quot;linear&quot;</span>,
    optimizer=optimizer,
    num_warmup_steps=<span class="hljs-number">0</span>,
    num_training_steps=num_training_steps,
)`}}),ae=new A({props:{code:`from huggingface_hub import get_full_repo_name

model_name = "distilbert-base-uncased-finetuned-imdb-accelerate"
repo_name = get_full_repo_name(model_name)
repo_name`,highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> get_full_repo_name

model_name = <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-imdb-accelerate&quot;</span>
repo_name = get_full_repo_name(model_name)
repo_name`}}),Ye=new A({props:{code:"'lewtun/distilbert-base-uncased-finetuned-imdb-accelerate'",highlighted:'<span class="hljs-string">&#x27;lewtun/distilbert-base-uncased-finetuned-imdb-accelerate&#x27;</span>'}}),We=new A({props:{code:`from huggingface_hub import Repository

output_dir = model_name
repo = Repository(output_dir, clone_from=repo_name)`,highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> Repository

output_dir = model_name
repo = Repository(output_dir, clone_from=repo_name)`}}),zt=new A({props:{code:`from tqdm.auto import tqdm
import torch
import math

progress_bar = tqdm(range(num_training_steps))

for epoch in range(num_train_epochs):
    # Training
    model.train()
    for batch in train_dataloader:
        outputs = model(**batch)
        loss = outputs.loss
        accelerator.backward(loss)

        optimizer.step()
        lr_scheduler.step()
        optimizer.zero_grad()
        progress_bar.update(1)

    # Evaluation
    model.eval()
    losses = []
    for step, batch in enumerate(eval_dataloader):
        with torch.no_grad():
            outputs = model(**batch)

        loss = outputs.loss
        losses.append(accelerator.gather(loss.repeat(batch_size)))

    losses = torch.cat(losses)
    losses = losses[: len(eval_dataset)]
    try:
        perplexity = math.exp(torch.mean(losses))
    except OverflowError:
        perplexity = float("inf")

    print(f">>> Epoch {epoch}: Perplexity: {perplexity}")

    # Save and upload
    accelerator.wait_for_everyone()
    unwrapped_model = accelerator.unwrap_model(model)
    unwrapped_model.save_pretrained(output_dir, save_function=accelerator.save)
    if accelerator.is_main_process:
        tokenizer.save_pretrained(output_dir)
        repo.push_to_hub(
            commit_message=f"Training in progress epoch {epoch}", blocking=False
        )`,highlighted:`<span class="hljs-keyword">from</span> tqdm.auto <span class="hljs-keyword">import</span> tqdm
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> math

progress_bar = tqdm(<span class="hljs-built_in">range</span>(num_training_steps))

<span class="hljs-keyword">for</span> epoch <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(num_train_epochs):
    <span class="hljs-comment"># Training</span>
    model.train()
    <span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> train_dataloader:
        outputs = model(**batch)
        loss = outputs.loss
        accelerator.backward(loss)

        optimizer.step()
        lr_scheduler.step()
        optimizer.zero_grad()
        progress_bar.update(<span class="hljs-number">1</span>)

    <span class="hljs-comment"># Evaluation</span>
    model.<span class="hljs-built_in">eval</span>()
    losses = []
    <span class="hljs-keyword">for</span> step, batch <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(eval_dataloader):
        <span class="hljs-keyword">with</span> torch.no_grad():
            outputs = model(**batch)

        loss = outputs.loss
        losses.append(accelerator.gather(loss.repeat(batch_size)))

    losses = torch.cat(losses)
    losses = losses[: <span class="hljs-built_in">len</span>(eval_dataset)]
    <span class="hljs-keyword">try</span>:
        perplexity = math.exp(torch.mean(losses))
    <span class="hljs-keyword">except</span> OverflowError:
        perplexity = <span class="hljs-built_in">float</span>(<span class="hljs-string">&quot;inf&quot;</span>)

    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&gt;&gt;&gt; Epoch <span class="hljs-subst">{epoch}</span>: Perplexity: <span class="hljs-subst">{perplexity}</span>&quot;</span>)

    <span class="hljs-comment"># Save and upload</span>
    accelerator.wait_for_everyone()
    unwrapped_model = accelerator.unwrap_model(model)
    unwrapped_model.save_pretrained(output_dir, save_function=accelerator.save)
    <span class="hljs-keyword">if</span> accelerator.is_main_process:
        tokenizer.save_pretrained(output_dir)
        repo.push_to_hub(
            commit_message=<span class="hljs-string">f&quot;Training in progress epoch <span class="hljs-subst">{epoch}</span>&quot;</span>, blocking=<span class="hljs-literal">False</span>
        )`}}),At=new A({props:{code:`Epoch 0: Perplexity: 11.397545307900472
Epoch 1: Perplexity: 10.904909330983092
Epoch 2: Perplexity: 10.729503505340409`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>Epoch <span class="hljs-number">0</span>: Perplexity: <span class="hljs-number">11.397545307900472</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>Epoch <span class="hljs-number">1</span>: Perplexity: <span class="hljs-number">10.904909330983092</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>Epoch <span class="hljs-number">2</span>: Perplexity: <span class="hljs-number">10.729503505340409</span>`}}),{c(){l=i("p"),f=a("In our use case we didn\u2019t need to do anything special with the training loop, but in some cases you might need to implement some custom logic. For these applications, you can use \u{1F917} Accelerate \u2014 let\u2019s take a look!"),d=u(),b=i("h2"),z=i("a"),k=i("span"),$(y.$$.fragment),q=u(),g=i("span"),D=a("Fine-tuning DistilBERT with \u{1F917} Accelerate"),F=u(),O=i("p"),M=a("As we saw with the "),N=i("code"),P=a("Trainer"),W=a(", fine-tuning a masked language model is very similar to the text classification example from "),B=i("a"),J=a("Chapter 3"),v=a(". In fact, the only subtlety is the use of a special data collator, and we\u2019ve already covered that earlier in this section!"),R=u(),U=i("p"),Y=a("However, we saw that "),X=i("code"),ue=a("DataCollatorForLanguageModeling"),Ae=a(" also applies random masking with each evaluation, so we\u2019ll see some fluctuations in our perplexity scores with each training run. One way to eliminate this source of randomness is to apply the masking "),ne=i("em"),le=a("once"),V=a(" on the whole test set, and then use the default data collator in \u{1F917} Transformers to collect the batches during evaluation. To see how this works, let\u2019s implement a simple function that applies the masking on a batch, similar to our first encounter with "),_e=i("code"),H=a("DataCollatorForLanguageModeling"),ce=a(":"),te=u(),$(ye.$$.fragment),qe=u(),Q=i("p"),De=a("Next, we\u2019ll apply this function to our test set and drop the unmasked columns so we can replace them with the masked ones. You can use whole word masking by replacing the "),Ke=i("code"),ie=a("data_collator"),Be=a(" above with the appropriate one, in which case you should remove the first line here:"),ge=u(),$(Me.$$.fragment),Je=u(),re=i("p"),pe=a("We can then set up the dataloaders as usual, but we\u2019ll use the "),Qe=i("code"),je=a("default_data_collator"),Re=a(" from \u{1F917} Transformers for the evaluation set:"),Xe=u(),$(we.$$.fragment),Z=u(),xe=i("p"),ee=a("Form here, we follow the standard steps with \u{1F917} Accelerate. The first order of business is to load a fresh version of the pretrained model:"),He=u(),$(se.$$.fragment),me=u(),ke=i("p"),ve=a("Then we need to specify the optimizer; we\u2019ll use the standard "),Fe=i("code"),$e=a("AdamW"),Ie=a(":"),C=u(),$(I.$$.fragment),Ge=u(),fe=i("p"),rt=a("With these objects, we can now prepare everything for training with the "),oe=i("code"),at=a("Accelerator"),Ne=a(" object:"),Ee=u(),$(S.$$.fragment),G=u(),he=i("p"),de=a("Now that our model, optimizer, and dataloaders are configured, we can specify the learning rate scheduler as follows:"),Pe=u(),$(Ue.$$.fragment),pt=u(),Te=i("p"),Ce=a("There is just one last thing to do before training: create a model repository on the Hugging Face Hub! We can use the \u{1F917} Hub library to first generate the full name of our repo:"),ht=u(),$(ae.$$.fragment),ot=u(),$(Ye.$$.fragment),Et=u(),nt=i("p"),Tt=a("then create and clone the repository using the "),ze=i("code"),be=a("Repository"),_t=a(" class from \u{1F917} Hub:"),dt=u(),$(We.$$.fragment),gt=u(),Ve=i("p"),ds=a("With that done, it\u2019s just a simple matter of writing out the full training and evaluation loop:"),Se=u(),$(zt.$$.fragment),Wt=u(),$(At.$$.fragment),cs=u(),ct=i("p"),Is=a("Cool, we\u2019ve been able to evaluate perplexity with each epoch and ensure that multiple training runs are reproducible!"),this.h()},l(c){l=r(c,"P",{});var L=h(l);f=o(L,"In our use case we didn\u2019t need to do anything special with the training loop, but in some cases you might need to implement some custom logic. For these applications, you can use \u{1F917} Accelerate \u2014 let\u2019s take a look!"),L.forEach(s),d=m(c),b=r(c,"H2",{class:!0});var ms=h(b);z=r(ms,"A",{id:!0,class:!0,href:!0});var fs=h(z);k=r(fs,"SPAN",{});var eo=h(k);j(y.$$.fragment,eo),eo.forEach(s),fs.forEach(s),q=m(ms),g=r(ms,"SPAN",{});var to=h(g);D=o(to,"Fine-tuning DistilBERT with \u{1F917} Accelerate"),to.forEach(s),ms.forEach(s),F=m(c),O=r(c,"P",{});var qt=h(O);M=o(qt,"As we saw with the "),N=r(qt,"CODE",{});var Bt=h(N);P=o(Bt,"Trainer"),Bt.forEach(s),W=o(qt,", fine-tuning a masked language model is very similar to the text classification example from "),B=r(qt,"A",{href:!0});var Gs=h(B);J=o(Gs,"Chapter 3"),Gs.forEach(s),v=o(qt,". In fact, the only subtlety is the use of a special data collator, and we\u2019ve already covered that earlier in this section!"),qt.forEach(s),R=m(c),U=r(c,"P",{});var lt=h(U);Y=o(lt,"However, we saw that "),X=r(lt,"CODE",{});var so=h(X);ue=o(so,"DataCollatorForLanguageModeling"),so.forEach(s),Ae=o(lt," also applies random masking with each evaluation, so we\u2019ll see some fluctuations in our perplexity scores with each training run. One way to eliminate this source of randomness is to apply the masking "),ne=r(lt,"EM",{});var Us=h(ne);le=o(Us,"once"),Us.forEach(s),V=o(lt," on the whole test set, and then use the default data collator in \u{1F917} Transformers to collect the batches during evaluation. To see how this works, let\u2019s implement a simple function that applies the masking on a batch, similar to our first encounter with "),_e=r(lt,"CODE",{});var ut=h(_e);H=o(ut,"DataCollatorForLanguageModeling"),ut.forEach(s),ce=o(lt,":"),lt.forEach(s),te=m(c),j(ye.$$.fragment,c),qe=m(c),Q=r(c,"P",{});var Ze=h(Q);De=o(Ze,"Next, we\u2019ll apply this function to our test set and drop the unmasked columns so we can replace them with the masked ones. You can use whole word masking by replacing the "),Ke=r(Ze,"CODE",{});var os=h(Ke);ie=o(os,"data_collator"),os.forEach(s),Be=o(Ze," above with the appropriate one, in which case you should remove the first line here:"),Ze.forEach(s),ge=m(c),j(Me.$$.fragment,c),Je=m(c),re=r(c,"P",{});var wt=h(re);pe=o(wt,"We can then set up the dataloaders as usual, but we\u2019ll use the "),Qe=r(wt,"CODE",{});var Ys=h(Qe);je=o(Ys,"default_data_collator"),Ys.forEach(s),Re=o(wt," from \u{1F917} Transformers for the evaluation set:"),wt.forEach(s),Xe=m(c),j(we.$$.fragment,c),Z=m(c),xe=r(c,"P",{});var ns=h(xe);ee=o(ns,"Form here, we follow the standard steps with \u{1F917} Accelerate. The first order of business is to load a fresh version of the pretrained model:"),ns.forEach(s),He=m(c),j(se.$$.fragment,c),me=m(c),ke=r(c,"P",{});var _s=h(ke);ve=o(_s,"Then we need to specify the optimizer; we\u2019ll use the standard "),Fe=r(_s,"CODE",{});var Vs=h(Fe);$e=o(Vs,"AdamW"),Vs.forEach(s),Ie=o(_s,":"),_s.forEach(s),C=m(c),j(I.$$.fragment,c),Ge=m(c),fe=r(c,"P",{});var mt=h(fe);rt=o(mt,"With these objects, we can now prepare everything for training with the "),oe=r(mt,"CODE",{});var Dt=h(oe);at=o(Dt,"Accelerator"),Dt.forEach(s),Ne=o(mt," object:"),mt.forEach(s),Ee=m(c),j(S.$$.fragment,c),G=m(c),he=r(c,"P",{});var gs=h(he);de=o(gs,"Now that our model, optimizer, and dataloaders are configured, we can specify the learning rate scheduler as follows:"),gs.forEach(s),Pe=m(c),j(Ue.$$.fragment,c),pt=m(c),Te=r(c,"P",{});var Ht=h(Te);Ce=o(Ht,"There is just one last thing to do before training: create a model repository on the Hugging Face Hub! We can use the \u{1F917} Hub library to first generate the full name of our repo:"),Ht.forEach(s),ht=m(c),j(ae.$$.fragment,c),ot=m(c),j(Ye.$$.fragment,c),Et=m(c),nt=r(c,"P",{});var ws=h(nt);Tt=o(ws,"then create and clone the repository using the "),ze=r(ws,"CODE",{});var ks=h(ze);be=o(ks,"Repository"),ks.forEach(s),_t=o(ws," class from \u{1F917} Hub:"),ws.forEach(s),dt=m(c),j(We.$$.fragment,c),gt=m(c),Ve=r(c,"P",{});var ao=h(Ve);ds=o(ao,"With that done, it\u2019s just a simple matter of writing out the full training and evaluation loop:"),ao.forEach(s),Se=m(c),j(zt.$$.fragment,c),Wt=m(c),j(At.$$.fragment,c),cs=m(c),ct=r(c,"P",{});var Js=h(ct);Is=o(Js,"Cool, we\u2019ve been able to evaluate perplexity with each epoch and ensure that multiple training runs are reproducible!"),Js.forEach(s),this.h()},h(){T(z,"id","fine-tuning-distilbert-with-accelerate"),T(z,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),T(z,"href","#fine-tuning-distilbert-with-accelerate"),T(b,"class","relative group"),T(B,"href","/course/chapter3")},m(c,L){p(c,l,L),t(l,f),p(c,d,L),p(c,b,L),t(b,z),t(z,k),x(y,k,null),t(b,q),t(b,g),t(g,D),p(c,F,L),p(c,O,L),t(O,M),t(O,N),t(N,P),t(O,W),t(O,B),t(B,J),t(O,v),p(c,R,L),p(c,U,L),t(U,Y),t(U,X),t(X,ue),t(U,Ae),t(U,ne),t(ne,le),t(U,V),t(U,_e),t(_e,H),t(U,ce),p(c,te,L),x(ye,c,L),p(c,qe,L),p(c,Q,L),t(Q,De),t(Q,Ke),t(Ke,ie),t(Q,Be),p(c,ge,L),x(Me,c,L),p(c,Je,L),p(c,re,L),t(re,pe),t(re,Qe),t(Qe,je),t(re,Re),p(c,Xe,L),x(we,c,L),p(c,Z,L),p(c,xe,L),t(xe,ee),p(c,He,L),x(se,c,L),p(c,me,L),p(c,ke,L),t(ke,ve),t(ke,Fe),t(Fe,$e),t(ke,Ie),p(c,C,L),x(I,c,L),p(c,Ge,L),p(c,fe,L),t(fe,rt),t(fe,oe),t(oe,at),t(fe,Ne),p(c,Ee,L),x(S,c,L),p(c,G,L),p(c,he,L),t(he,de),p(c,Pe,L),x(Ue,c,L),p(c,pt,L),p(c,Te,L),t(Te,Ce),p(c,ht,L),x(ae,c,L),p(c,ot,L),x(Ye,c,L),p(c,Et,L),p(c,nt,L),t(nt,Tt),t(nt,ze),t(ze,be),t(nt,_t),p(c,dt,L),x(We,c,L),p(c,gt,L),p(c,Ve,L),t(Ve,ds),p(c,Se,L),x(zt,c,L),p(c,Wt,L),x(At,c,L),p(c,cs,L),p(c,ct,L),t(ct,Is),us=!0},i(c){us||(_(y.$$.fragment,c),_(ye.$$.fragment,c),_(Me.$$.fragment,c),_(we.$$.fragment,c),_(se.$$.fragment,c),_(I.$$.fragment,c),_(S.$$.fragment,c),_(Ue.$$.fragment,c),_(ae.$$.fragment,c),_(Ye.$$.fragment,c),_(We.$$.fragment,c),_(zt.$$.fragment,c),_(At.$$.fragment,c),us=!0)},o(c){w(y.$$.fragment,c),w(ye.$$.fragment,c),w(Me.$$.fragment,c),w(we.$$.fragment,c),w(se.$$.fragment,c),w(I.$$.fragment,c),w(S.$$.fragment,c),w(Ue.$$.fragment,c),w(ae.$$.fragment,c),w(Ye.$$.fragment,c),w(We.$$.fragment,c),w(zt.$$.fragment,c),w(At.$$.fragment,c),us=!1},d(c){c&&s(l),c&&s(d),c&&s(b),E(y),c&&s(F),c&&s(O),c&&s(R),c&&s(U),c&&s(te),E(ye,c),c&&s(qe),c&&s(Q),c&&s(ge),E(Me,c),c&&s(Je),c&&s(re),c&&s(Xe),E(we,c),c&&s(Z),c&&s(xe),c&&s(He),E(se,c),c&&s(me),c&&s(ke),c&&s(C),E(I,c),c&&s(Ge),c&&s(fe),c&&s(Ee),E(S,c),c&&s(G),c&&s(he),c&&s(Pe),E(Ue,c),c&&s(pt),c&&s(Te),c&&s(ht),E(ae,c),c&&s(ot),E(Ye,c),c&&s(Et),c&&s(nt),c&&s(dt),E(We,c),c&&s(gt),c&&s(Ve),c&&s(Se),E(zt,c),c&&s(Wt),E(At,c),c&&s(cs),c&&s(ct)}}}function Fu(K){let l,f,d,b,z,k,y,q;return{c(){l=i("p"),f=a("\u270F\uFE0F "),d=i("strong"),b=a("Try it out!"),z=a(" To quantify the benefits of domain adaptation, fine-tune a classifier on the IMDb labels for both the pretrained and fine-tuned DistilBERT checkpoints. If you need a refresher on text classification, check out "),k=i("a"),y=a("Chapter 3"),q=a("."),this.h()},l(g){l=r(g,"P",{});var D=h(l);f=o(D,"\u270F\uFE0F "),d=r(D,"STRONG",{});var F=h(d);b=o(F,"Try it out!"),F.forEach(s),z=o(D," To quantify the benefits of domain adaptation, fine-tune a classifier on the IMDb labels for both the pretrained and fine-tuned DistilBERT checkpoints. If you need a refresher on text classification, check out "),k=r(D,"A",{href:!0});var O=h(k);y=o(O,"Chapter 3"),O.forEach(s),q=o(D,"."),D.forEach(s),this.h()},h(){T(k,"href","/course/chapter3")},m(g,D){p(g,l,D),t(l,f),t(l,d),t(d,b),t(l,z),t(l,k),t(k,y),t(l,q)},d(g){g&&s(l)}}}function Nu(K){let l,f,d,b,z,k,y,q,g,D,F,O,M,N,P,W,B,J,v,R,U,Y,X,ue,Ae,ne,le,V,_e,H,ce,te,ye,qe,Q,De,Ke,ie,Be,ge,Me,Je,re,pe,Qe,je,Re,Xe,we,Z,xe,ee,He,se,me,ke,ve,Fe,$e,Ie,C,I,Ge,fe,rt,oe,at,Ne,Ee,S,G,he,de,Pe,Ue,pt,Te,Ce,ht,ae,ot,Ye,Et,nt,Tt,ze,be,_t,dt,We,gt,Ve,ds,Se,zt,Wt,At,cs,ct,Is,us,c,L,ms,fs,eo,to,qt,Bt,Gs,lt,so,Us,ut,Ze,os,wt,Ys,ns,_s,Vs,mt,Dt,gs,Ht,ws,ks,ao,Js,It,tr,Qs,sr,ar,Vo,or,nr,nl,Xs,ll,Zs,il,et,lr,Jo,ir,rr,Qo,pr,hr,Xo,dr,cr,Zo,ur,mr,en,fr,_r,rl,ea,pl,ta,hl,Gt,gr,tn,wr,kr,sn,br,yr,dl,bs,cl,ys,vr,oo,$r,jr,ul,ls,vs,an,sa,xr,on,Er,ml,aa,fl,no,Tr,_l,tt,zr,nn,Ar,qr,ln,Dr,Mr,lo,Pr,Cr,rn,Sr,Or,pn,Lr,Kr,gl,oa,wl,na,kl,kt,Rr,hn,Fr,Nr,dn,Wr,Br,cn,Hr,Ir,bl,$s,Gr,un,Ur,Yr,yl,la,vl,ia,$l,js,Vr,mn,Jr,Qr,jl,xs,xl,io,Xr,El,ra,Tl,Es,zl,ro,Zr,Al,pa,ql,ha,Dl,po,ep,Ml,da,Pl,ca,Cl,Ut,tp,fn,sp,ap,_n,op,np,Sl,ua,Ol,ma,Ll,ho,lp,Kl,Ts,fa,ip,gn,rp,pp,hp,_a,dp,wn,cp,up,Rl,co,mp,Fl,ga,Nl,ft,fp,kn,_p,gp,bn,wp,kp,yn,bp,yp,vn,vp,$p,Wl,Yt,jp,$n,xp,Ep,jn,Tp,zp,Bl,wa,Hl,ka,Il,st,Ap,xn,qp,Dp,En,Mp,Pp,Tn,Cp,Sp,zn,Op,Lp,An,Kp,Rp,Gl,ba,Ul,ya,Yl,uo,Fp,Vl,va,Jl,$a,Ql,bt,Np,qn,Wp,Bp,Dn,Hp,Ip,Mn,Gp,Up,Xl,is,zs,Pn,ja,Yp,xa,Vp,Cn,Jp,Qp,Zl,yt,Xp,mo,Zp,eh,Sn,th,sh,On,ah,oh,ei,Ea,ti,vt,nh,Ln,lh,ih,Kn,rh,ph,Rn,hh,dh,si,Ta,ai,za,oi,Vt,ch,Fn,uh,mh,Nn,fh,_h,ni,As,li,fo,Jt,gh,Wn,wh,kh,Bn,bh,yh,ii,Mt,Pt,_o,go,vh,ri,Aa,pi,qa,hi,qs,di,Qt,$h,Hn,jh,xh,wo,Eh,Th,ci,Da,ui,Ma,mi,Xt,zh,In,Ah,qh,Gn,Dh,Mh,fi,Pa,_i,ko,Ph,gi,Ca,wi,bo,Ch,ki,Ct,St,yo,rs,Ds,Un,Sa,Sh,Yn,Oh,bi,Oa,yi,vo,Lh,vi,Ot,Lt,$o,La,$i,jo,Kh,ji,Kt,Rt,xo,Eo,Rh,xi,Ft,Nt,To,Ka,Ei,zo,Fh,Ti,Ao,Ms,zi,qo,ps,Ps,Vn,Ra,Nh,Jn,Wh,Ai,Zt,Bh,Qn,Hh,Ih,Xn,Gh,Uh,qi,Fa,Di,Do,Yh,Mi,Na,Pi,Wa,Ci,Mo,Vh,Si,Ba,Oi,Cs,Jh,Po,Qh,Xh,Li,Ss,Ki;d=new _u({props:{fw:K[0]}}),q=new Hs({});const Zh=[wu,gu],Ha=[];function ed(e,n){return e[0]==="pt"?0:1}M=ed(K),N=Ha[M]=Zh[M](K),Z=new er({props:{id:"mqElG5QJWUg"}}),ee=new Bs({props:{$$slots:{default:[ku]},$$scope:{ctx:K}}}),ve=new Hs({});const td=[yu,bu],Ia=[];function sd(e,n){return e[0]==="pt"?0:1}ze=sd(K),be=Ia[ze]=td[ze](K),Ve=new A({props:{code:'text = "This is a great [MASK]."',highlighted:'text = <span class="hljs-string">&quot;This is a great [MASK].&quot;</span>'}}),Bt=new A({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)`}});const ad=[$u,vu],Ga=[];function od(e,n){return e[0]==="pt"?0:1}ut=od(K),Ze=Ga[ut]=ad[ut](K),wt=new A({props:{code:`'>>> This is a great deal.'
'>>> This is a great success.'
'>>> This is a great adventure.'
'>>> This is a great idea.'
'>>> This is a great feat.'`,highlighted:`<span class="hljs-string">&#x27;&gt;&gt;&gt; This is a great deal.&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; This is a great success.&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; This is a great adventure.&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; This is a great idea.&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; This is a great feat.&#x27;</span>`}}),Ht=new Hs({}),Xs=new A({props:{code:`from datasets import load_dataset

imdb_dataset = load_dataset("imdb")
imdb_dataset`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

imdb_dataset = load_dataset(<span class="hljs-string">&quot;imdb&quot;</span>)
imdb_dataset`}}),Zs=new A({props:{code:`DatasetDict({
    train: Dataset({
        features: ['text', 'label'],
        num_rows: 25000
    })
    test: Dataset({
        features: ['text', 'label'],
        num_rows: 25000
    })
    unsupervised: Dataset({
        features: ['text', 'label'],
        num_rows: 50000
    })
})`,highlighted:`DatasetDict({
    train: Dataset({
        features: [<span class="hljs-string">&#x27;text&#x27;</span>, <span class="hljs-string">&#x27;label&#x27;</span>],
        num_rows: <span class="hljs-number">25000</span>
    })
    test: Dataset({
        features: [<span class="hljs-string">&#x27;text&#x27;</span>, <span class="hljs-string">&#x27;label&#x27;</span>],
        num_rows: <span class="hljs-number">25000</span>
    })
    unsupervised: Dataset({
        features: [<span class="hljs-string">&#x27;text&#x27;</span>, <span class="hljs-string">&#x27;label&#x27;</span>],
        num_rows: <span class="hljs-number">50000</span>
    })
})`}}),ea=new A({props:{code:`sample = imdb_dataset["train"].shuffle(seed=42).select(range(3))

for row in sample:
    print(f"\\n'>>> Review: {row['text']}'")
    print(f"'>>> Label: {row['label']}'")`,highlighted:`sample = imdb_dataset[<span class="hljs-string">&quot;train&quot;</span>].shuffle(seed=<span class="hljs-number">42</span>).select(<span class="hljs-built_in">range</span>(<span class="hljs-number">3</span>))

<span class="hljs-keyword">for</span> row <span class="hljs-keyword">in</span> sample:
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;\\n&#x27;&gt;&gt;&gt; Review: <span class="hljs-subst">{row[<span class="hljs-string">&#x27;text&#x27;</span>]}</span>&#x27;&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&#x27;&gt;&gt;&gt; Label: <span class="hljs-subst">{row[<span class="hljs-string">&#x27;label&#x27;</span>]}</span>&#x27;&quot;</span>)`}}),ta=new A({props:{code:`
'>>> Review: This is your typical Priyadarshan movie--a bunch of loony characters out on some silly mission. His signature climax has the entire cast of the film coming together and fighting each other in some crazy moshpit over hidden money. Whether it is a winning lottery ticket in Malamaal Weekly, black money in Hera Pheri, "kodokoo" in Phir Hera Pheri, etc., etc., the director is becoming ridiculously predictable. Don\\'t get me wrong; as clich\xE9d and preposterous his movies may be, I usually end up enjoying the comedy. However, in most his previous movies there has actually been some good humor, (Hungama and Hera Pheri being noteworthy ones). Now, the hilarity of his films is fading as he is using the same formula over and over again.<br /><br />Songs are good. Tanushree Datta looks awesome. Rajpal Yadav is irritating, and Tusshar is not a whole lot better. Kunal Khemu is OK, and Sharman Joshi is the best.'
'>>> Label: 0'

'>>> Review: Okay, the story makes no sense, the characters lack any dimensionally, the best dialogue is ad-libs about the low quality of movie, the cinematography is dismal, and only editing saves a bit of the muddle, but Sam" Peckinpah directed the film. Somehow, his direction is not enough. For those who appreciate Peckinpah and his great work, this movie is a disappointment. Even a great cast cannot redeem the time the viewer wastes with this minimal effort.<br /><br />The proper response to the movie is the contempt that the director San Peckinpah, James Caan, Robert Duvall, Burt Young, Bo Hopkins, Arthur Hill, and even Gig Young bring to their work. Watch the great Peckinpah films. Skip this mess.'
'>>> Label: 0'

'>>> Review: I saw this movie at the theaters when I was about 6 or 7 years old. I loved it then, and have recently come to own a VHS version. <br /><br />My 4 and 6 year old children love this movie and have been asking again and again to watch it. <br /><br />I have enjoyed watching it again too. Though I have to admit it is not as good on a little TV.<br /><br />I do not have older children so I do not know what they would think of it. <br /><br />The songs are very cute. My daughter keeps singing them over and over.<br /><br />Hope this helps.'
'>>> Label: 1'`,highlighted:`
<span class="hljs-string">&#x27;&gt;&gt;&gt; Review: This is your typical Priyadarshan movie--a bunch of loony characters out on some silly mission. His signature climax has the entire cast of the film coming together and fighting each other in some crazy moshpit over hidden money. Whether it is a winning lottery ticket in Malamaal Weekly, black money in Hera Pheri, &quot;kodokoo&quot; in Phir Hera Pheri, etc., etc., the director is becoming ridiculously predictable. Don\\&#x27;t get me wrong; as clich\xE9d and preposterous his movies may be, I usually end up enjoying the comedy. However, in most his previous movies there has actually been some good humor, (Hungama and Hera Pheri being noteworthy ones). Now, the hilarity of his films is fading as he is using the same formula over and over again.&lt;br /&gt;&lt;br /&gt;Songs are good. Tanushree Datta looks awesome. Rajpal Yadav is irritating, and Tusshar is not a whole lot better. Kunal Khemu is OK, and Sharman Joshi is the best.&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; Label: 0&#x27;</span>

<span class="hljs-string">&#x27;&gt;&gt;&gt; Review: Okay, the story makes no sense, the characters lack any dimensionally, the best dialogue is ad-libs about the low quality of movie, the cinematography is dismal, and only editing saves a bit of the muddle, but Sam&quot; Peckinpah directed the film. Somehow, his direction is not enough. For those who appreciate Peckinpah and his great work, this movie is a disappointment. Even a great cast cannot redeem the time the viewer wastes with this minimal effort.&lt;br /&gt;&lt;br /&gt;The proper response to the movie is the contempt that the director San Peckinpah, James Caan, Robert Duvall, Burt Young, Bo Hopkins, Arthur Hill, and even Gig Young bring to their work. Watch the great Peckinpah films. Skip this mess.&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; Label: 0&#x27;</span>

<span class="hljs-string">&#x27;&gt;&gt;&gt; Review: I saw this movie at the theaters when I was about 6 or 7 years old. I loved it then, and have recently come to own a VHS version. &lt;br /&gt;&lt;br /&gt;My 4 and 6 year old children love this movie and have been asking again and again to watch it. &lt;br /&gt;&lt;br /&gt;I have enjoyed watching it again too. Though I have to admit it is not as good on a little TV.&lt;br /&gt;&lt;br /&gt;I do not have older children so I do not know what they would think of it. &lt;br /&gt;&lt;br /&gt;The songs are very cute. My daughter keeps singing them over and over.&lt;br /&gt;&lt;br /&gt;Hope this helps.&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; Label: 1&#x27;</span>`}}),bs=new Bs({props:{$$slots:{default:[ju]},$$scope:{ctx:K}}}),sa=new Hs({}),aa=new er({props:{id:"8PmhEIXhBvI"}}),oa=new A({props:{code:`def tokenize_function(examples):
    result = tokenizer(examples["text"])
    if tokenizer.is_fast:
        result["word_ids"] = [result.word_ids(i) for i in range(len(result["input_ids"]))]
    return result


# Use batched=True to activate fast multithreading!
tokenized_datasets = imdb_dataset.map(
    tokenize_function, batched=True, remove_columns=["text", "label"]
)
tokenized_datasets`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize_function</span>(<span class="hljs-params">examples</span>):
    result = tokenizer(examples[<span class="hljs-string">&quot;text&quot;</span>])
    <span class="hljs-keyword">if</span> tokenizer.is_fast:
        result[<span class="hljs-string">&quot;word_ids&quot;</span>] = [result.word_ids(i) <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(result[<span class="hljs-string">&quot;input_ids&quot;</span>]))]
    <span class="hljs-keyword">return</span> result


<span class="hljs-comment"># Use batched=True to activate fast multithreading!</span>
tokenized_datasets = imdb_dataset.<span class="hljs-built_in">map</span>(
    tokenize_function, batched=<span class="hljs-literal">True</span>, remove_columns=[<span class="hljs-string">&quot;text&quot;</span>, <span class="hljs-string">&quot;label&quot;</span>]
)
tokenized_datasets`}}),na=new A({props:{code:`DatasetDict({
    train: Dataset({
        features: ['attention_mask', 'input_ids', 'word_ids'],
        num_rows: 25000
    })
    test: Dataset({
        features: ['attention_mask', 'input_ids', 'word_ids'],
        num_rows: 25000
    })
    unsupervised: Dataset({
        features: ['attention_mask', 'input_ids', 'word_ids'],
        num_rows: 50000
    })
})`,highlighted:`DatasetDict({
    train: Dataset({
        features: [<span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;word_ids&#x27;</span>],
        num_rows: <span class="hljs-number">25000</span>
    })
    test: Dataset({
        features: [<span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;word_ids&#x27;</span>],
        num_rows: <span class="hljs-number">25000</span>
    })
    unsupervised: Dataset({
        features: [<span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;word_ids&#x27;</span>],
        num_rows: <span class="hljs-number">50000</span>
    })
})`}}),la=new A({props:{code:"tokenizer.model_max_length",highlighted:"tokenizer.model_max_length"}}),ia=new A({props:{code:"512",highlighted:'<span class="hljs-number">512</span>'}}),xs=new Bs({props:{$$slots:{default:[xu]},$$scope:{ctx:K}}}),ra=new A({props:{code:"chunk_size = 128",highlighted:'chunk_size = <span class="hljs-number">128</span>'}}),Es=new Bs({props:{warning:!0,$$slots:{default:[Eu]},$$scope:{ctx:K}}}),pa=new A({props:{code:`# Slicing produces a list of lists for each feature
tokenized_samples = tokenized_datasets["train"][:3]

for idx, sample in enumerate(tokenized_samples["input_ids"]):
    print(f"'>>> Review {idx} length: {len(sample)}'")`,highlighted:`<span class="hljs-comment"># Slicing produces a list of lists for each feature</span>
tokenized_samples = tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>][:<span class="hljs-number">3</span>]

<span class="hljs-keyword">for</span> idx, sample <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(tokenized_samples[<span class="hljs-string">&quot;input_ids&quot;</span>]):
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&#x27;&gt;&gt;&gt; Review <span class="hljs-subst">{idx}</span> length: <span class="hljs-subst">{<span class="hljs-built_in">len</span>(sample)}</span>&#x27;&quot;</span>)`}}),ha=new A({props:{code:`'>>> Review 0 length: 200'
'>>> Review 1 length: 559'
'>>> Review 2 length: 192'`,highlighted:`<span class="hljs-string">&#x27;&gt;&gt;&gt; Review 0 length: 200&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; Review 1 length: 559&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; Review 2 length: 192&#x27;</span>`}}),da=new A({props:{code:`concatenated_examples = {
    k: sum(tokenized_samples[k], []) for k in tokenized_samples.keys()
}
total_length = len(concatenated_examples["input_ids"])
print(f"'>>> Concatenated reviews length: {total_length}'")`,highlighted:`concatenated_examples = {
    k: <span class="hljs-built_in">sum</span>(tokenized_samples[k], []) <span class="hljs-keyword">for</span> k <span class="hljs-keyword">in</span> tokenized_samples.keys()
}
total_length = <span class="hljs-built_in">len</span>(concatenated_examples[<span class="hljs-string">&quot;input_ids&quot;</span>])
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&#x27;&gt;&gt;&gt; Concatenated reviews length: <span class="hljs-subst">{total_length}</span>&#x27;&quot;</span>)`}}),ca=new A({props:{code:"'>>> Concatenated reviews length: 951'",highlighted:'<span class="hljs-string">&#x27;&gt;&gt;&gt; Concatenated reviews length: 951&#x27;</span>'}}),ua=new A({props:{code:`chunks = {
    k: [t[i : i + chunk_size] for i in range(0, total_length, chunk_size)]
    for k, t in concatenated_examples.items()
}

for chunk in chunks["input_ids"]:
    print(f"'>>> Chunk length: {len(chunk)}'")`,highlighted:`chunks = {
    k: [t[i : i + chunk_size] <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">0</span>, total_length, chunk_size)]
    <span class="hljs-keyword">for</span> k, t <span class="hljs-keyword">in</span> concatenated_examples.items()
}

<span class="hljs-keyword">for</span> chunk <span class="hljs-keyword">in</span> chunks[<span class="hljs-string">&quot;input_ids&quot;</span>]:
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&#x27;&gt;&gt;&gt; Chunk length: <span class="hljs-subst">{<span class="hljs-built_in">len</span>(chunk)}</span>&#x27;&quot;</span>)`}}),ma=new A({props:{code:`'>>> Chunk length: 128'
'>>> Chunk length: 128'
'>>> Chunk length: 128'
'>>> Chunk length: 128'
'>>> Chunk length: 128'
'>>> Chunk length: 128'
'>>> Chunk length: 128'
'>>> Chunk length: 55'`,highlighted:`<span class="hljs-string">&#x27;&gt;&gt;&gt; Chunk length: 128&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; Chunk length: 128&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; Chunk length: 128&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; Chunk length: 128&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; Chunk length: 128&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; Chunk length: 128&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; Chunk length: 128&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; Chunk length: 55&#x27;</span>`}}),ga=new A({props:{code:`def group_texts(examples):
    # Concatenate all texts
    concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()}
    # Compute length of concatenated texts
    total_length = len(concatenated_examples[list(examples.keys())[0]])
    # We drop the last chunk if it's smaller than chunk_size
    total_length = (total_length // chunk_size) * chunk_size
    # Split by chunks of max_len
    result = {
        k: [t[i : i + chunk_size] for i in range(0, total_length, chunk_size)]
        for k, t in concatenated_examples.items()
    }
    # Create a new labels column
    result["labels"] = result["input_ids"].copy()
    return result`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">group_texts</span>(<span class="hljs-params">examples</span>):
    <span class="hljs-comment"># Concatenate all texts</span>
    concatenated_examples = {k: <span class="hljs-built_in">sum</span>(examples[k], []) <span class="hljs-keyword">for</span> k <span class="hljs-keyword">in</span> examples.keys()}
    <span class="hljs-comment"># Compute length of concatenated texts</span>
    total_length = <span class="hljs-built_in">len</span>(concatenated_examples[<span class="hljs-built_in">list</span>(examples.keys())[<span class="hljs-number">0</span>]])
    <span class="hljs-comment"># We drop the last chunk if it&#x27;s smaller than chunk_size</span>
    total_length = (total_length // chunk_size) * chunk_size
    <span class="hljs-comment"># Split by chunks of max_len</span>
    result = {
        k: [t[i : i + chunk_size] <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">0</span>, total_length, chunk_size)]
        <span class="hljs-keyword">for</span> k, t <span class="hljs-keyword">in</span> concatenated_examples.items()
    }
    <span class="hljs-comment"># Create a new labels column</span>
    result[<span class="hljs-string">&quot;labels&quot;</span>] = result[<span class="hljs-string">&quot;input_ids&quot;</span>].copy()
    <span class="hljs-keyword">return</span> result`}}),wa=new A({props:{code:`lm_datasets = tokenized_datasets.map(group_texts, batched=True)
lm_datasets`,highlighted:`lm_datasets = tokenized_datasets.<span class="hljs-built_in">map</span>(group_texts, batched=<span class="hljs-literal">True</span>)
lm_datasets`}}),ka=new A({props:{code:`DatasetDict({
    train: Dataset({
        features: ['attention_mask', 'input_ids', 'labels', 'word_ids'],
        num_rows: 61289
    })
    test: Dataset({
        features: ['attention_mask', 'input_ids', 'labels', 'word_ids'],
        num_rows: 59905
    })
    unsupervised: Dataset({
        features: ['attention_mask', 'input_ids', 'labels', 'word_ids'],
        num_rows: 122963
    })
})`,highlighted:`DatasetDict({
    train: Dataset({
        features: [<span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;labels&#x27;</span>, <span class="hljs-string">&#x27;word_ids&#x27;</span>],
        num_rows: <span class="hljs-number">61289</span>
    })
    test: Dataset({
        features: [<span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;labels&#x27;</span>, <span class="hljs-string">&#x27;word_ids&#x27;</span>],
        num_rows: <span class="hljs-number">59905</span>
    })
    unsupervised: Dataset({
        features: [<span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;labels&#x27;</span>, <span class="hljs-string">&#x27;word_ids&#x27;</span>],
        num_rows: <span class="hljs-number">122963</span>
    })
})`}}),ba=new A({props:{code:'tokenizer.decode(lm_datasets["train"][1]["input_ids"])',highlighted:'tokenizer.decode(lm_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">1</span>][<span class="hljs-string">&quot;input_ids&quot;</span>])'}}),ya=new A({props:{code:`".... at.......... high. a classic line : inspector : i'm here to sack one of your teachers. student : welcome to bromwell high. i expect that many adults of my age think that bromwell high is far fetched. what a pity that it isn't! [SEP] [CLS] homelessness ( or houselessness as george carlin stated ) has been an issue for years but never a plan to help those on the street that were once considered human who did everything from going to school, work, or vote for the matter. most people think of the homeless"`,highlighted:'<span class="hljs-string">&quot;.... at.......... high. a classic line : inspector : i&#x27;m here to sack one of your teachers. student : welcome to bromwell high. i expect that many adults of my age think that bromwell high is far fetched. what a pity that it isn&#x27;t! [SEP] [CLS] homelessness ( or houselessness as george carlin stated ) has been an issue for years but never a plan to help those on the street that were once considered human who did everything from going to school, work, or vote for the matter. most people think of the homeless&quot;</span>'}}),va=new A({props:{code:'tokenizer.decode(lm_datasets["train"][1]["labels"])',highlighted:'tokenizer.decode(lm_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">1</span>][<span class="hljs-string">&quot;labels&quot;</span>])'}}),$a=new A({props:{code:`".... at.......... high. a classic line : inspector : i'm here to sack one of your teachers. student : welcome to bromwell high. i expect that many adults of my age think that bromwell high is far fetched. what a pity that it isn't! [SEP] [CLS] homelessness ( or houselessness as george carlin stated ) has been an issue for years but never a plan to help those on the street that were once considered human who did everything from going to school, work, or vote for the matter. most people think of the homeless"`,highlighted:'<span class="hljs-string">&quot;.... at.......... high. a classic line : inspector : i&#x27;m here to sack one of your teachers. student : welcome to bromwell high. i expect that many adults of my age think that bromwell high is far fetched. what a pity that it isn&#x27;t! [SEP] [CLS] homelessness ( or houselessness as george carlin stated ) has been an issue for years but never a plan to help those on the street that were once considered human who did everything from going to school, work, or vote for the matter. most people think of the homeless&quot;</span>'}}),ja=new Hs({}),Ea=new A({props:{code:`from transformers import DataCollatorForLanguageModeling

data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm_probability=0.15)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForLanguageModeling

data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm_probability=<span class="hljs-number">0.15</span>)`}}),Ta=new A({props:{code:`samples = [lm_datasets["train"][i] for i in range(2)]
for sample in samples:
    _ = sample.pop("word_ids")

for chunk in data_collator(samples)["input_ids"]:
    print(f"\\n'>>> {tokenizer.decode(chunk)}'")`,highlighted:`samples = [lm_datasets[<span class="hljs-string">&quot;train&quot;</span>][i] <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">2</span>)]
<span class="hljs-keyword">for</span> sample <span class="hljs-keyword">in</span> samples:
    _ = sample.pop(<span class="hljs-string">&quot;word_ids&quot;</span>)

<span class="hljs-keyword">for</span> chunk <span class="hljs-keyword">in</span> data_collator(samples)[<span class="hljs-string">&quot;input_ids&quot;</span>]:
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;\\n&#x27;&gt;&gt;&gt; <span class="hljs-subst">{tokenizer.decode(chunk)}</span>&#x27;&quot;</span>)`}}),za=new A({props:{code:`'>>> [CLS] bromwell [MASK] is a cartoon comedy. it ran at the same [MASK] as some other [MASK] about school life, [MASK] as " teachers ". [MASK] [MASK] [MASK] in the teaching [MASK] lead [MASK] to believe that bromwell high\\'[MASK] satire is much closer to reality than is " teachers ". the scramble [MASK] [MASK] financially, the [MASK]ful students whogn [MASK] right through [MASK] pathetic teachers\\'pomp, the pettiness of the whole situation, distinction remind me of the schools i knew and their students. when i saw [MASK] episode in [MASK] a student repeatedly tried to burn down the school, [MASK] immediately recalled. [MASK]...'

'>>> .... at.. [MASK]... [MASK]... high. a classic line plucked inspector : i\\'[MASK] here to [MASK] one of your [MASK]. student : welcome to bromwell [MASK]. i expect that many adults of my age think that [MASK]mwell [MASK] is [MASK] fetched. what a pity that it isn\\'t! [SEP] [CLS] [MASK]ness ( or [MASK]lessness as george \u5B87in stated )\u516C been an issue for years but never [MASK] plan to help those on the street that were once considered human [MASK] did everything from going to school, [MASK], [MASK] vote for the matter. most people think [MASK] the homeless'`,highlighted:`<span class="hljs-string">&#x27;&gt;&gt;&gt; [CLS] bromwell [MASK] is a cartoon comedy. it ran at the same [MASK] as some other [MASK] about school life, [MASK] as &quot; teachers &quot;. [MASK] [MASK] [MASK] in the teaching [MASK] lead [MASK] to believe that bromwell high\\&#x27;[MASK] satire is much closer to reality than is &quot; teachers &quot;. the scramble [MASK] [MASK] financially, the [MASK]ful students whogn [MASK] right through [MASK] pathetic teachers\\&#x27;pomp, the pettiness of the whole situation, distinction remind me of the schools i knew and their students. when i saw [MASK] episode in [MASK] a student repeatedly tried to burn down the school, [MASK] immediately recalled. [MASK]...&#x27;</span>

<span class="hljs-string">&#x27;&gt;&gt;&gt; .... at.. [MASK]... [MASK]... high. a classic line plucked inspector : i\\&#x27;[MASK] here to [MASK] one of your [MASK]. student : welcome to bromwell [MASK]. i expect that many adults of my age think that [MASK]mwell [MASK] is [MASK] fetched. what a pity that it isn\\&#x27;t! [SEP] [CLS] [MASK]ness ( or [MASK]lessness as george \u5B87in stated )\u516C been an issue for years but never [MASK] plan to help those on the street that were once considered human [MASK] did everything from going to school, [MASK], [MASK] vote for the matter. most people think [MASK] the homeless&#x27;</span>`}}),As=new Bs({props:{$$slots:{default:[Tu]},$$scope:{ctx:K}}});let it=K[0]==="pt"&&iu();const nd=[Au,zu],Ua=[];function ld(e,n){return e[0]==="pt"?0:1}Mt=ld(K),Pt=Ua[Mt]=nd[Mt](K),Aa=new A({props:{code:`samples = [lm_datasets["train"][i] for i in range(2)]
batch = whole_word_masking_data_collator(samples)

for chunk in batch["input_ids"]:
    print(f"\\n'>>> {tokenizer.decode(chunk)}'")`,highlighted:`samples = [lm_datasets[<span class="hljs-string">&quot;train&quot;</span>][i] <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">2</span>)]
batch = whole_word_masking_data_collator(samples)

<span class="hljs-keyword">for</span> chunk <span class="hljs-keyword">in</span> batch[<span class="hljs-string">&quot;input_ids&quot;</span>]:
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;\\n&#x27;&gt;&gt;&gt; <span class="hljs-subst">{tokenizer.decode(chunk)}</span>&#x27;&quot;</span>)`}}),qa=new A({props:{code:`'>>> [CLS] bromwell high is a cartoon comedy [MASK] it ran at the same time as some other programs about school life, such as " teachers ". my 35 years in the teaching profession lead me to believe that bromwell high\\'s satire is much closer to reality than is " teachers ". the scramble to survive financially, the insightful students who can see right through their pathetic teachers\\'pomp, the pettiness of the whole situation, all remind me of the schools i knew and their students. when i saw the episode in which a student repeatedly tried to burn down the school, i immediately recalled.....'

'>>> .... [MASK] [MASK] [MASK] [MASK]....... high. a classic line : inspector : i\\'m here to sack one of your teachers. student : welcome to bromwell high. i expect that many adults of my age think that bromwell high is far fetched. what a pity that it isn\\'t! [SEP] [CLS] homelessness ( or houselessness as george carlin stated ) has been an issue for years but never a plan to help those on the street that were once considered human who did everything from going to school, work, or vote for the matter. most people think of the homeless'`,highlighted:`<span class="hljs-string">&#x27;&gt;&gt;&gt; [CLS] bromwell high is a cartoon comedy [MASK] it ran at the same time as some other programs about school life, such as &quot; teachers &quot;. my 35 years in the teaching profession lead me to believe that bromwell high\\&#x27;s satire is much closer to reality than is &quot; teachers &quot;. the scramble to survive financially, the insightful students who can see right through their pathetic teachers\\&#x27;pomp, the pettiness of the whole situation, all remind me of the schools i knew and their students. when i saw the episode in which a student repeatedly tried to burn down the school, i immediately recalled.....&#x27;</span>

<span class="hljs-string">&#x27;&gt;&gt;&gt; .... [MASK] [MASK] [MASK] [MASK]....... high. a classic line : inspector : i\\&#x27;m here to sack one of your teachers. student : welcome to bromwell high. i expect that many adults of my age think that bromwell high is far fetched. what a pity that it isn\\&#x27;t! [SEP] [CLS] homelessness ( or houselessness as george carlin stated ) has been an issue for years but never a plan to help those on the street that were once considered human who did everything from going to school, work, or vote for the matter. most people think of the homeless&#x27;</span>`}}),qs=new Bs({props:{$$slots:{default:[qu]},$$scope:{ctx:K}}}),Da=new A({props:{code:`train_size = 10_000
test_size = int(0.1 * train_size)

downsampled_dataset = lm_datasets["train"].train_test_split(
    train_size=train_size, test_size=test_size, seed=42
)
downsampled_dataset`,highlighted:`train_size = <span class="hljs-number">10_000</span>
test_size = <span class="hljs-built_in">int</span>(<span class="hljs-number">0.1</span> * train_size)

downsampled_dataset = lm_datasets[<span class="hljs-string">&quot;train&quot;</span>].train_test_split(
    train_size=train_size, test_size=test_size, seed=<span class="hljs-number">42</span>
)
downsampled_dataset`}}),Ma=new A({props:{code:`DatasetDict({
    train: Dataset({
        features: ['attention_mask', 'input_ids', 'labels', 'word_ids'],
        num_rows: 10000
    })
    test: Dataset({
        features: ['attention_mask', 'input_ids', 'labels', 'word_ids'],
        num_rows: 1000
    })
})`,highlighted:`DatasetDict({
    train: Dataset({
        features: [<span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;labels&#x27;</span>, <span class="hljs-string">&#x27;word_ids&#x27;</span>],
        num_rows: <span class="hljs-number">10000</span>
    })
    test: Dataset({
        features: [<span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;labels&#x27;</span>, <span class="hljs-string">&#x27;word_ids&#x27;</span>],
        num_rows: <span class="hljs-number">1000</span>
    })
})`}}),Pa=new A({props:{code:`from huggingface_hub import notebook_login

notebook_login()`,highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

notebook_login()`}}),Ca=new A({props:{code:"huggingface-cli login",highlighted:'huggingface-<span class="hljs-keyword">cli</span> login'}});const id=[Mu,Du],Ya=[];function rd(e,n){return e[0]==="tf"?0:1}Ct=rd(K),St=Ya[Ct]=id[Ct](K),Sa=new Hs({}),Oa=new er({props:{id:"NURcDHhYe98"}});const pd=[Cu,Pu],Va=[];function hd(e,n){return e[0]==="pt"?0:1}Ot=hd(K),Lt=Va[Ot]=pd[Ot](K),La=new A({props:{code:"Perplexity: 21.75",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>Perplexity: <span class="hljs-number">21.75</span>'}});const dd=[Ou,Su],Ja=[];function cd(e,n){return e[0]==="pt"?0:1}Kt=cd(K),Rt=Ja[Kt]=dd[Kt](K);const ud=[Ku,Lu],Qa=[];function md(e,n){return e[0]==="pt"?0:1}Ft=md(K),Nt=Qa[Ft]=ud[Ft](K),Ka=new A({props:{code:"Perplexity: 11.32",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>Perplexity: <span class="hljs-number">11.32</span>'}});let Oe=K[0]==="pt"&&ru();Ms=new Bs({props:{$$slots:{default:[Ru]},$$scope:{ctx:K}}});let Le=K[0]==="pt"&&pu();return Ra=new Hs({}),Fa=new A({props:{code:`from transformers import pipeline

mask_filler = pipeline(
    "fill-mask", model="huggingface-course/distilbert-base-uncased-finetuned-imdb"
)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

mask_filler = pipeline(
    <span class="hljs-string">&quot;fill-mask&quot;</span>, model=<span class="hljs-string">&quot;huggingface-course/distilbert-base-uncased-finetuned-imdb&quot;</span>
)`}}),Na=new A({props:{code:`preds = mask_filler(text)

for pred in preds:
    print(f">>> {pred['sequence']}")`,highlighted:`preds = mask_filler(text)

<span class="hljs-keyword">for</span> pred <span class="hljs-keyword">in</span> preds:
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&gt;&gt;&gt; <span class="hljs-subst">{pred[<span class="hljs-string">&#x27;sequence&#x27;</span>]}</span>&quot;</span>)`}}),Wa=new A({props:{code:`'>>> this is a great movie.'
'>>> this is a great film.'
'>>> this is a great story.'
'>>> this is a great movies.'
'>>> this is a great character.'`,highlighted:`<span class="hljs-string">&#x27;&gt;&gt;&gt; this is a great movie.&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; this is a great film.&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; this is a great story.&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; this is a great movies.&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt;&gt; this is a great character.&#x27;</span>`}}),Ba=new er({props:{id:"0Oxphw4Q9fo"}}),Ss=new Bs({props:{$$slots:{default:[Fu]},$$scope:{ctx:K}}}),{c(){l=i("meta"),f=u(),$(d.$$.fragment),b=u(),z=i("h1"),k=i("a"),y=i("span"),$(q.$$.fragment),g=u(),D=i("span"),F=a("Fine-tuning a masked language model"),O=u(),N.c(),P=u(),W=i("p"),B=a("For many NLP applications involving Transformer models, you can simply take a pretrained model from the Hugging Face Hub and fine-tune it directly on your data for the task at hand. Provided that the corpus used for pretraining is not too different from the corpus used for fine-tuning, transfer learning will usually produce good results."),J=u(),v=i("p"),R=a("However, there are a few cases where you\u2019ll want to first fine-tune the language models on your data, before training a task-specific head. For example, if your dataset contains legal contracts or scientific articles, a vanilla Transformer model like BERT will typically treat the domain-specific words in your corpus as rare tokens, and the resulting performance may be less than satisfactory. By fine-tuning the language model on in-domain data you can boost the performance of many downstream tasks, which means you usually only have to do this step once!"),U=u(),Y=i("p"),X=a("This process of fine-tuning a pretrained language model on in-domain data is usually called "),ue=i("em"),Ae=a("domain adaptation"),ne=a(". It was popularized in 2018 by "),le=i("a"),V=a("ULMFiT"),_e=a(", which was one of the first neural architectures (based on LSTMs) to make transfer learning really work for NLP. An example of domain adaptation with ULMFiT is shown in the image below; in this section we\u2019ll do something similar, but with a Transformer instead of an LSTM!"),H=u(),ce=i("div"),te=i("img"),qe=u(),Q=i("img"),Ke=u(),ie=i("p"),Be=a("By the end of this section you\u2019ll have a "),ge=i("a"),Me=a("masked language model"),Je=a(" on the Hub that can autocomplete sentences as shown below:"),re=u(),pe=i("iframe"),je=u(),Re=i("p"),Xe=a("Let\u2019s dive in!"),we=u(),$(Z.$$.fragment),xe=u(),$(ee.$$.fragment),He=u(),se=i("h2"),me=i("a"),ke=i("span"),$(ve.$$.fragment),Fe=u(),$e=i("span"),Ie=a("Picking a pretrained model for masked language modeling"),C=u(),I=i("p"),Ge=a("To get started, let\u2019s pick a suitable pretrained model for masked language modeling. As shown in the following screenshot, you can find a list of candidates by applying the \u201CFill-Mask\u201D filter on the "),fe=i("a"),rt=a("Hugging Face Hub"),oe=a(":"),at=u(),Ne=i("div"),Ee=i("img"),G=u(),he=i("p"),de=a("Although the BERT and RoBERTa family of models are the most downloaded, we\u2019ll use a model called "),Pe=i("a"),Ue=a("DistilBERT"),pt=a(`
that can be trained much faster with little to no loss in downstream performance. This model was trained using a special technique called `),Te=i("a"),Ce=i("em"),ht=a("knowledge distillation"),ae=a(", where a large \u201Cteacher model\u201D like BERT is used to guide the training of a \u201Cstudent model\u201D that has far fewer parameters. An explanation of the details of knowledge distillation would take us too far afield in this section, but if you\u2019re interested you can read all about it in "),ot=i("a"),Ye=i("em"),Et=a("Natural Language Processing with Transformers"),nt=a(" (colloquially known as the Transformers textbook)."),Tt=u(),be.c(),_t=u(),dt=i("p"),We=a("With around 67 million parameters, DistilBERT is approximately two times smaller than the BERT base model, which roughly translates into a two-fold speedup in training \u2014 nice! Let\u2019s now see what kinds of tokens this model predicts are the most likely completions of a small sample of text:"),gt=u(),$(Ve.$$.fragment),ds=u(),Se=i("p"),zt=a("As humans, we can imagine many possibilities for the "),Wt=i("code"),At=a("[MASK]"),cs=a(" token, such as \u201Cday\u201D, \u201Cride\u201D, or \u201Cpainting\u201D. For pretrained models, the predictions depend on the corpus the model was trained on, since it learns to pick up the statistical patterns present in the data. Like BERT, DistilBERT was pretrained on the "),ct=i("a"),Is=a("English Wikipedia"),us=a(" and "),c=i("a"),L=a("BookCorpus"),ms=a(" datasets, so we expect the predictions for "),fs=i("code"),eo=a("[MASK]"),to=a(" to reflect these domains. To predict the mask we need DistilBERT\u2019s tokenizer to produce the inputs for the model, so let\u2019s download that from the Hub as well:"),qt=u(),$(Bt.$$.fragment),Gs=u(),lt=i("p"),so=a("With a tokenizer and a model, we can now pass our text example to the model, extract the logits, and print out the top 5 candidates:"),Us=u(),Ze.c(),os=u(),$(wt.$$.fragment),Ys=u(),ns=i("p"),_s=a("We can see from the outputs that the model\u2019s predictions refer to everyday terms, which is perhaps not surprising given the foundation of English Wikipedia. Let\u2019s see how we can change this domain to something a bit more niche \u2014 highly polarized movie reviews!"),Vs=u(),mt=i("h2"),Dt=i("a"),gs=i("span"),$(Ht.$$.fragment),ws=u(),ks=i("span"),ao=a("The dataset"),Js=u(),It=i("p"),tr=a("To showcase domain adaptation, we\u2019ll use the famous "),Qs=i("a"),sr=a("Large Movie Review Dataset"),ar=a(" (or IMDb for short), which is a corpus of movie reviews that is often used to benchmark sentiment analysis models. By fine-tuning DistilBERT on this corpus, we expect the language model will adapt its vocabulary from the factual data of Wikipedia that it was pretrained on to the more subjective elements of movie reviews. We can get the data from the Hugging Face Hub with the "),Vo=i("code"),or=a("load_dataset()"),nr=a(" function from \u{1F917} Datasets:"),nl=u(),$(Xs.$$.fragment),ll=u(),$(Zs.$$.fragment),il=u(),et=i("p"),lr=a("We can see that the "),Jo=i("code"),ir=a("train"),rr=a(" and "),Qo=i("code"),pr=a("test"),hr=a(" splits each consist of 25,000 reviews, while there is an unlabeled split called "),Xo=i("code"),dr=a("unsupervised"),cr=a(" that contains 50,000 reviews. Let\u2019s take a look at a few samples to get an idea of what kind of text we\u2019re dealing with. As we\u2019ve done in previous chapters of the course, we\u2019ll chain the "),Zo=i("code"),ur=a("Dataset.shuffle()"),mr=a(" and "),en=i("code"),fr=a("Dataset.select()"),_r=a(" functions to create a random sample:"),rl=u(),$(ea.$$.fragment),pl=u(),$(ta.$$.fragment),hl=u(),Gt=i("p"),gr=a("Yep, these are certainly movie reviews, and if you\u2019re old enough you may even understand the comment in the last review about owning a VHS version \u{1F61C}! Although we won\u2019t need the labels for language modeling, we can already see that a "),tn=i("code"),wr=a("0"),kr=a(" denotes a negative review, while a "),sn=i("code"),br=a("1"),yr=a(" corresponds to a positive one."),dl=u(),$(bs.$$.fragment),cl=u(),ys=i("p"),vr=a("Now that we\u2019ve had a quick look at the data, let\u2019s dive into preparing it for masked language modeling. As we\u2019ll see, there are some additional steps that one needs to take compared to the sequence classification tasks we saw in "),oo=i("a"),$r=a("Chapter 3"),jr=a(". Let\u2019s go!"),ul=u(),ls=i("h2"),vs=i("a"),an=i("span"),$(sa.$$.fragment),xr=u(),on=i("span"),Er=a("Preprocessing the data"),ml=u(),$(aa.$$.fragment),fl=u(),no=i("p"),Tr=a("For both auto-regressive and masked language modeling, a common preprocessing step is to concatenate all the examples and then split the whole corpus into chunks of equal size. This is quite different from our usual approach, where we simply tokenize individual examples. Why concatenate everything together? The reason is that individual examples might get truncated if they\u2019re too long, and that would result in losing information that might be useful for the language modeling task!"),_l=u(),tt=i("p"),zr=a("So to get started, we\u2019ll first tokenize our corpus as usual, but "),nn=i("em"),Ar=a("without"),qr=a(" setting the "),ln=i("code"),Dr=a("truncation=True"),Mr=a(" option in our tokenizer. We\u2019ll also grab the word IDs if they are available ((which they will be if we\u2019re using a fast tokenizer, as described in "),lo=i("a"),Pr=a("Chapter 6"),Cr=a("), as we will need them later on to do whole word masking. We\u2019ll wrap this in a simple function, and while we\u2019re at it we\u2019ll remove the "),rn=i("code"),Sr=a("text"),Or=a(" and "),pn=i("code"),Lr=a("label"),Kr=a(" columns since we don\u2019t need them any longer:"),gl=u(),$(oa.$$.fragment),wl=u(),$(na.$$.fragment),kl=u(),kt=i("p"),Rr=a("Since DistilBERT is a BERT-like model, we can see that the encoded texts consist of the "),hn=i("code"),Fr=a("input_ids"),Nr=a(" and "),dn=i("code"),Wr=a("attention_mask"),Br=a(" that we\u2019ve seen in other chapters, as well as the "),cn=i("code"),Hr=a("word_ids"),Ir=a(" we added."),bl=u(),$s=i("p"),Gr=a("Now that we\u2019ve tokenized our movie reviews, the next step is to group them all together and split the result into chunks. But how big should these chunks be? This will ultimately be determined by the amount of GPU memory that you have available, but a good starting point is to see what the model\u2019s maximum context size is. This can be inferred by inspecting the "),un=i("code"),Ur=a("model_max_length"),Yr=a(" attribute of the tokenizer:"),yl=u(),$(la.$$.fragment),vl=u(),$(ia.$$.fragment),$l=u(),js=i("p"),Vr=a("This value is derived from the "),mn=i("em"),Jr=a("tokenizer_config.json"),Qr=a(" file associated with a checkpoint; in this case we can see that the context size is 512 tokens, just like with BERT."),jl=u(),$(xs.$$.fragment),xl=u(),io=i("p"),Xr=a("So, in order to run our experiments on GPUs like those found on Google Colab, we\u2019ll pick something a bit smaller that can fit in memory:"),El=u(),$(ra.$$.fragment),Tl=u(),$(Es.$$.fragment),zl=u(),ro=i("p"),Zr=a("Now comes the fun part. To show how the concatenation works, let\u2019s take a few reviews from our tokenized training set and print out the number of tokens per review:"),Al=u(),$(pa.$$.fragment),ql=u(),$(ha.$$.fragment),Dl=u(),po=i("p"),ep=a("We can then concatenate all these examples with a simple dictionary comprehension, as follows:"),Ml=u(),$(da.$$.fragment),Pl=u(),$(ca.$$.fragment),Cl=u(),Ut=i("p"),tp=a("Great, the total length checks out \u2014 so now let\u2019s split the concatenated reviews into chunks of the size given by "),fn=i("code"),sp=a("block_size"),ap=a(". To do so, we iterate over the features in "),_n=i("code"),op=a("concatenated_examples"),np=a(" and use a list comprehension to create slices of each feature. The result is a dictionary of chunks for each feature:"),Sl=u(),$(ua.$$.fragment),Ol=u(),$(ma.$$.fragment),Ll=u(),ho=i("p"),lp=a("As you can see in this example, the last chunk will generally be smaller than the maximum chunk size. There are two main strategies for dealing with this:"),Kl=u(),Ts=i("ul"),fa=i("li"),ip=a("Drop the last chunk if it\u2019s smaller than "),gn=i("code"),rp=a("chunk_size"),pp=a("."),hp=u(),_a=i("li"),dp=a("Pad the last chunk until its length equals "),wn=i("code"),cp=a("chunk_size"),up=a("."),Rl=u(),co=i("p"),mp=a("We\u2019ll take the first approach here, so let\u2019s wrap all of the above logic in a single function that we can apply to our tokenized datasets:"),Fl=u(),$(ga.$$.fragment),Nl=u(),ft=i("p"),fp=a("Note that in the last step of "),kn=i("code"),_p=a("group_texts()"),gp=a(" we create a new "),bn=i("code"),wp=a("labels"),kp=a(" column which is a copy of the "),yn=i("code"),bp=a("input_ids"),yp=a(" one. As we\u2019ll see shortly, that\u2019s because in masked language modeling the objective is to predict randomly masked tokens in the input batch, and by creating a "),vn=i("code"),vp=a("labels"),$p=a(" column we provide the ground truth for our language model to learn from."),Wl=u(),Yt=i("p"),jp=a("Let\u2019s now apply "),$n=i("code"),xp=a("group_texts()"),Ep=a(" to our tokenized datasets using our trusty "),jn=i("code"),Tp=a("Dataset.map()"),zp=a(" function:"),Bl=u(),$(wa.$$.fragment),Hl=u(),$(ka.$$.fragment),Il=u(),st=i("p"),Ap=a("You can see that grouping and then chunking the texts has produced many more examples than our original 25,000 for the "),xn=i("code"),qp=a("train"),Dp=a(" and "),En=i("code"),Mp=a("test"),Pp=a(" splits. That\u2019s because we now have examples involving "),Tn=i("em"),Cp=a("contiguous tokens"),Sp=a(" that span across multiple examples from the original corpus. You can see this explicitly by looking for the special "),zn=i("code"),Op=a("[SEP]"),Lp=a(" and "),An=i("code"),Kp=a("[CLS]"),Rp=a(" tokens in one of the chunks:"),Gl=u(),$(ba.$$.fragment),Ul=u(),$(ya.$$.fragment),Yl=u(),uo=i("p"),Fp=a("In this example you can see two overlapping movie reviews, one about a high school movie and the other about homelessness. Let\u2019s also check out what the labels look like for masked language modeling:"),Vl=u(),$(va.$$.fragment),Jl=u(),$($a.$$.fragment),Ql=u(),bt=i("p"),Np=a("As expected from our "),qn=i("code"),Wp=a("group_texts()"),Bp=a(" function above, this looks identical to the decoded "),Dn=i("code"),Hp=a("input_ids"),Ip=a(" \u2014 but then how can our model possibly learn anything? We\u2019re missing a key step: inserting "),Mn=i("code"),Gp=a("[MASK]"),Up=a(" tokens at random positions in the inputs! Let\u2019s see how we can do this on the fly during fine-tuning using a special data collator."),Xl=u(),is=i("h2"),zs=i("a"),Pn=i("span"),$(ja.$$.fragment),Yp=u(),xa=i("span"),Vp=a("Fine-tuning DistilBERT with the "),Cn=i("code"),Jp=a("Trainer"),Qp=a(" API"),Zl=u(),yt=i("p"),Xp=a("Fine-tuning a masked language model is almost identical to fine-tuning a sequence classification model, like we did in "),mo=i("a"),Zp=a("Chapter 3"),eh=a(". The only difference is that we need a special data collator that can randomly mask some of the tokens in each batch of texts. Fortunately, \u{1F917} Transformers comes prepared with a dedicated "),Sn=i("code"),th=a("DataCollatorForLanguageModeling"),sh=a(" for just this task. We just have to pass it the tokenizer and an "),On=i("code"),ah=a("mlm_probability"),oh=a(" argument that specifies what fraction of the tokens to mask. We\u2019ll pick 15%, which is the amount used for BERT and a common choice in the literature:"),ei=u(),$(Ea.$$.fragment),ti=u(),vt=i("p"),nh=a("To see how the random masking works, let\u2019s feed a few examples to the data collator. Since it expects a list of "),Ln=i("code"),lh=a("dict"),ih=a("s, where each "),Kn=i("code"),rh=a("dict"),ph=a(" represents a single chunk of contiguous text, we first iterate over the dataset before feeding the batch to the collator. We remove the "),Rn=i("code"),hh=a('"word_ids"'),dh=a(" key for this data collator as it does not expect it:"),si=u(),$(Ta.$$.fragment),ai=u(),$(za.$$.fragment),oi=u(),Vt=i("p"),ch=a("Nice, it worked! We can see that the "),Fn=i("code"),uh=a("[MASK]"),mh=a(" token has been randomly inserted at various locations in our text. These will be the tokens which our model will have to predict during training \u2014 and the beauty of the data collator is that it will randomize the "),Nn=i("code"),fh=a("[MASK]"),_h=a(" insertion with every batch!"),ni=u(),$(As.$$.fragment),li=u(),it&&it.c(),fo=u(),Jt=i("p"),gh=a("When training models for masked language modeling, one technique that can be used is to mask whole words together, not just individual tokens. This approach is called "),Wn=i("em"),wh=a("whole word masking"),kh=a(". If we want to use whole word masking, we will need to build a data collator ourselves. A data collator is just a function that takes a list of samples and converts them into a batch, so let\u2019s do this now! We\u2019ll use the word IDs computed earlier to make a map between word indices and the corresponding tokens, then randomly decide which words to mask and apply that mask on the inputs. Note that the labels are all "),Bn=i("code"),bh=a("-100"),yh=a(" except for the ones corresponding to mask words."),ii=u(),Pt.c(),_o=u(),go=i("p"),vh=a("Next, we can try it on the same samples as before:"),ri=u(),$(Aa.$$.fragment),pi=u(),$(qa.$$.fragment),hi=u(),$(qs.$$.fragment),di=u(),Qt=i("p"),$h=a("Now that we have two data collators, the rest of the fine-tuning steps are standard. Training can take a while on Google Colab if you\u2019re not lucky enough to score a mythical P100 GPU \u{1F62D}, so we\u2019ll first downsample the size of the training set to a few thousand examples. Don\u2019t worry, we\u2019ll still get a pretty decent language model! A quick way to downsample a dataset in \u{1F917} Datasets is via the "),Hn=i("code"),jh=a("Dataset.train_test_split()"),xh=a(" function that we saw in "),wo=i("a"),Eh=a("Chapter 5"),Th=a(":"),ci=u(),$(Da.$$.fragment),ui=u(),$(Ma.$$.fragment),mi=u(),Xt=i("p"),zh=a("This has automatically created new "),In=i("code"),Ah=a("train"),qh=a(" and "),Gn=i("code"),Dh=a("test"),Mh=a(" splits, with the training set size set to 10,000 examples and the validation set to 10% of that \u2014 feel free to increase this if you have a beefy GPU! The next thing we need to do is log in to the Hugging Face Hub. If you\u2019re running this code in a notebook, you can do so with the following utility function:"),fi=u(),$(Pa.$$.fragment),_i=u(),ko=i("p"),Ph=a("which will display a widget where you can enter your credentials. Alternatively, you can run:"),gi=u(),$(Ca.$$.fragment),wi=u(),bo=i("p"),Ch=a("in your favorite terminal and log in there."),ki=u(),St.c(),yo=u(),rs=i("h3"),Ds=i("a"),Un=i("span"),$(Sa.$$.fragment),Sh=u(),Yn=i("span"),Oh=a("Perplexity for language models"),bi=u(),$(Oa.$$.fragment),yi=u(),vo=i("p"),Lh=a("Unlike other tasks like text classification or question answering where we\u2019re given a labeled corpus to train on, with language modeling we don\u2019t have any explicit labels. So how do we determine what makes a good language model? Like with the autocorrect feature in your phone, a good language model is one that assigns high probabilities to sentences that are grammatically correct, and low probabilities to nonsense sentences. To give you a better idea of what this looks like, you can find whole sets of \u201Cautocorrect fails\u201D online, where the model in a person\u2019s phone has produced some rather funny (and often inappropriate) completions!"),vi=u(),Lt.c(),$o=u(),$(La.$$.fragment),$i=u(),jo=i("p"),Kh=a("A lower perplexity score means a better language model, and we can see here that our starting model has a somewhat large value. Let\u2019s see if we can lower it by fine-tuning! To do that, we first run the training loop:"),ji=u(),Rt.c(),xo=u(),Eo=i("p"),Rh=a("and then compute the resulting perplexity on the test set as before:"),xi=u(),Nt.c(),To=u(),$(Ka.$$.fragment),Ei=u(),zo=i("p"),Fh=a("Nice \u2014 this is quite a reduction in perplexity, which tells us the model has learned something about the domain of movie reviews!"),Ti=u(),Oe&&Oe.c(),Ao=u(),$(Ms.$$.fragment),zi=u(),Le&&Le.c(),qo=u(),ps=i("h2"),Ps=i("a"),Vn=i("span"),$(Ra.$$.fragment),Nh=u(),Jn=i("span"),Wh=a("Using our fine-tuned model"),Ai=u(),Zt=i("p"),Bh=a("You can interact with your fine-tuned model either by using its widget on the Hub or locally with the "),Qn=i("code"),Hh=a("pipeline"),Ih=a(" from \u{1F917} Transformers. Let\u2019s use the latter to download our model using the "),Xn=i("code"),Gh=a("fill-mask"),Uh=a(" pipeline:"),qi=u(),$(Fa.$$.fragment),Di=u(),Do=i("p"),Yh=a("We can then feed the pipeline our sample text of \u201CThis is a great [MASK]\u201D and see what the top 5 predictions are:"),Mi=u(),$(Na.$$.fragment),Pi=u(),$(Wa.$$.fragment),Ci=u(),Mo=i("p"),Vh=a("Neat \u2014 our model has clearly adapted its weights to predict words that are more strongly associated with movies!"),Si=u(),$(Ba.$$.fragment),Oi=u(),Cs=i("p"),Jh=a("This wraps up our first experiment with training a language model. In "),Po=i("a"),Qh=a("section 6"),Xh=a(" you\u2019ll learn how to train an auto-regressive model like GPT-2 from scratch; head over there if you\u2019d like to see how you can pretrain your very own Transformer model!"),Li=u(),$(Ss.$$.fragment),this.h()},l(e){const n=mu('[data-svelte="svelte-1phssyn"]',document.head);l=r(n,"META",{name:!0,content:!0}),n.forEach(s),f=m(e),j(d.$$.fragment,e),b=m(e),z=r(e,"H1",{class:!0});var Xa=h(z);k=r(Xa,"A",{id:!0,class:!0,href:!0});var Co=h(k);y=r(Co,"SPAN",{});var Zn=h(y);j(q.$$.fragment,Zn),Zn.forEach(s),Co.forEach(s),g=m(Xa),D=r(Xa,"SPAN",{});var So=h(D);F=o(So,"Fine-tuning a masked language model"),So.forEach(s),Xa.forEach(s),O=m(e),N.l(e),P=m(e),W=r(e,"P",{});var Oo=h(W);B=o(Oo,"For many NLP applications involving Transformer models, you can simply take a pretrained model from the Hugging Face Hub and fine-tune it directly on your data for the task at hand. Provided that the corpus used for pretraining is not too different from the corpus used for fine-tuning, transfer learning will usually produce good results."),Oo.forEach(s),J=m(e),v=r(e,"P",{});var el=h(v);R=o(el,"However, there are a few cases where you\u2019ll want to first fine-tune the language models on your data, before training a task-specific head. For example, if your dataset contains legal contracts or scientific articles, a vanilla Transformer model like BERT will typically treat the domain-specific words in your corpus as rare tokens, and the resulting performance may be less than satisfactory. By fine-tuning the language model on in-domain data you can boost the performance of many downstream tasks, which means you usually only have to do this step once!"),el.forEach(s),U=m(e),Y=r(e,"P",{});var hs=h(Y);X=o(hs,"This process of fine-tuning a pretrained language model on in-domain data is usually called "),ue=r(hs,"EM",{});var tl=h(ue);Ae=o(tl,"domain adaptation"),tl.forEach(s),ne=o(hs,". It was popularized in 2018 by "),le=r(hs,"A",{href:!0,rel:!0});var sl=h(le);V=o(sl,"ULMFiT"),sl.forEach(s),_e=o(hs,", which was one of the first neural architectures (based on LSTMs) to make transfer learning really work for NLP. An example of domain adaptation with ULMFiT is shown in the image below; in this section we\u2019ll do something similar, but with a Transformer instead of an LSTM!"),hs.forEach(s),H=m(e),ce=r(e,"DIV",{class:!0});var Os=h(ce);te=r(Os,"IMG",{class:!0,src:!0,alt:!0}),qe=m(Os),Q=r(Os,"IMG",{class:!0,src:!0,alt:!0}),Os.forEach(s),Ke=m(e),ie=r(e,"P",{});var Za=h(ie);Be=o(Za,"By the end of this section you\u2019ll have a "),ge=r(Za,"A",{href:!0,rel:!0});var Lo=h(ge);Me=o(Lo,"masked language model"),Lo.forEach(s),Je=o(Za," on the Hub that can autocomplete sentences as shown below:"),Za.forEach(s),re=m(e),pe=r(e,"IFRAME",{src:!0,frameborder:!0,height:!0,title:!0,class:!0,allow:!0,sandbox:!0}),h(pe).forEach(s),je=m(e),Re=r(e,"P",{});var Ko=h(Re);Xe=o(Ko,"Let\u2019s dive in!"),Ko.forEach(s),we=m(e),j(Z.$$.fragment,e),xe=m(e),j(ee.$$.fragment,e),He=m(e),se=r(e,"H2",{class:!0});var Ls=h(se);me=r(Ls,"A",{id:!0,class:!0,href:!0});var Ro=h(me);ke=r(Ro,"SPAN",{});var al=h(ke);j(ve.$$.fragment,al),al.forEach(s),Ro.forEach(s),Fe=m(Ls),$e=r(Ls,"SPAN",{});var ol=h($e);Ie=o(ol,"Picking a pretrained model for masked language modeling"),ol.forEach(s),Ls.forEach(s),C=m(e),I=r(e,"P",{});var Ri=h(I);Ge=o(Ri,"To get started, let\u2019s pick a suitable pretrained model for masked language modeling. As shown in the following screenshot, you can find a list of candidates by applying the \u201CFill-Mask\u201D filter on the "),fe=r(Ri,"A",{href:!0,rel:!0});var fd=h(fe);rt=o(fd,"Hugging Face Hub"),fd.forEach(s),oe=o(Ri,":"),Ri.forEach(s),at=m(e),Ne=r(e,"DIV",{class:!0});var _d=h(Ne);Ee=r(_d,"IMG",{src:!0,alt:!0,width:!0}),_d.forEach(s),G=m(e),he=r(e,"P",{});var Ks=h(he);de=o(Ks,"Although the BERT and RoBERTa family of models are the most downloaded, we\u2019ll use a model called "),Pe=r(Ks,"A",{href:!0,rel:!0});var gd=h(Pe);Ue=o(gd,"DistilBERT"),gd.forEach(s),pt=o(Ks,`
that can be trained much faster with little to no loss in downstream performance. This model was trained using a special technique called `),Te=r(Ks,"A",{href:!0,rel:!0});var wd=h(Te);Ce=r(wd,"EM",{});var kd=h(Ce);ht=o(kd,"knowledge distillation"),kd.forEach(s),wd.forEach(s),ae=o(Ks,", where a large \u201Cteacher model\u201D like BERT is used to guide the training of a \u201Cstudent model\u201D that has far fewer parameters. An explanation of the details of knowledge distillation would take us too far afield in this section, but if you\u2019re interested you can read all about it in "),ot=r(Ks,"A",{href:!0,rel:!0});var bd=h(ot);Ye=r(bd,"EM",{});var yd=h(Ye);Et=o(yd,"Natural Language Processing with Transformers"),yd.forEach(s),bd.forEach(s),nt=o(Ks," (colloquially known as the Transformers textbook)."),Ks.forEach(s),Tt=m(e),be.l(e),_t=m(e),dt=r(e,"P",{});var vd=h(dt);We=o(vd,"With around 67 million parameters, DistilBERT is approximately two times smaller than the BERT base model, which roughly translates into a two-fold speedup in training \u2014 nice! Let\u2019s now see what kinds of tokens this model predicts are the most likely completions of a small sample of text:"),vd.forEach(s),gt=m(e),j(Ve.$$.fragment,e),ds=m(e),Se=r(e,"P",{});var es=h(Se);zt=o(es,"As humans, we can imagine many possibilities for the "),Wt=r(es,"CODE",{});var $d=h(Wt);At=o($d,"[MASK]"),$d.forEach(s),cs=o(es," token, such as \u201Cday\u201D, \u201Cride\u201D, or \u201Cpainting\u201D. For pretrained models, the predictions depend on the corpus the model was trained on, since it learns to pick up the statistical patterns present in the data. Like BERT, DistilBERT was pretrained on the "),ct=r(es,"A",{href:!0,rel:!0});var jd=h(ct);Is=o(jd,"English Wikipedia"),jd.forEach(s),us=o(es," and "),c=r(es,"A",{href:!0,rel:!0});var xd=h(c);L=o(xd,"BookCorpus"),xd.forEach(s),ms=o(es," datasets, so we expect the predictions for "),fs=r(es,"CODE",{});var Ed=h(fs);eo=o(Ed,"[MASK]"),Ed.forEach(s),to=o(es," to reflect these domains. To predict the mask we need DistilBERT\u2019s tokenizer to produce the inputs for the model, so let\u2019s download that from the Hub as well:"),es.forEach(s),qt=m(e),j(Bt.$$.fragment,e),Gs=m(e),lt=r(e,"P",{});var Td=h(lt);so=o(Td,"With a tokenizer and a model, we can now pass our text example to the model, extract the logits, and print out the top 5 candidates:"),Td.forEach(s),Us=m(e),Ze.l(e),os=m(e),j(wt.$$.fragment,e),Ys=m(e),ns=r(e,"P",{});var zd=h(ns);_s=o(zd,"We can see from the outputs that the model\u2019s predictions refer to everyday terms, which is perhaps not surprising given the foundation of English Wikipedia. Let\u2019s see how we can change this domain to something a bit more niche \u2014 highly polarized movie reviews!"),zd.forEach(s),Vs=m(e),mt=r(e,"H2",{class:!0});var Fi=h(mt);Dt=r(Fi,"A",{id:!0,class:!0,href:!0});var Ad=h(Dt);gs=r(Ad,"SPAN",{});var qd=h(gs);j(Ht.$$.fragment,qd),qd.forEach(s),Ad.forEach(s),ws=m(Fi),ks=r(Fi,"SPAN",{});var Dd=h(ks);ao=o(Dd,"The dataset"),Dd.forEach(s),Fi.forEach(s),Js=m(e),It=r(e,"P",{});var Fo=h(It);tr=o(Fo,"To showcase domain adaptation, we\u2019ll use the famous "),Qs=r(Fo,"A",{href:!0,rel:!0});var Md=h(Qs);sr=o(Md,"Large Movie Review Dataset"),Md.forEach(s),ar=o(Fo," (or IMDb for short), which is a corpus of movie reviews that is often used to benchmark sentiment analysis models. By fine-tuning DistilBERT on this corpus, we expect the language model will adapt its vocabulary from the factual data of Wikipedia that it was pretrained on to the more subjective elements of movie reviews. We can get the data from the Hugging Face Hub with the "),Vo=r(Fo,"CODE",{});var Pd=h(Vo);or=o(Pd,"load_dataset()"),Pd.forEach(s),nr=o(Fo," function from \u{1F917} Datasets:"),Fo.forEach(s),nl=m(e),j(Xs.$$.fragment,e),ll=m(e),j(Zs.$$.fragment,e),il=m(e),et=r(e,"P",{});var $t=h(et);lr=o($t,"We can see that the "),Jo=r($t,"CODE",{});var Cd=h(Jo);ir=o(Cd,"train"),Cd.forEach(s),rr=o($t," and "),Qo=r($t,"CODE",{});var Sd=h(Qo);pr=o(Sd,"test"),Sd.forEach(s),hr=o($t," splits each consist of 25,000 reviews, while there is an unlabeled split called "),Xo=r($t,"CODE",{});var Od=h(Xo);dr=o(Od,"unsupervised"),Od.forEach(s),cr=o($t," that contains 50,000 reviews. Let\u2019s take a look at a few samples to get an idea of what kind of text we\u2019re dealing with. As we\u2019ve done in previous chapters of the course, we\u2019ll chain the "),Zo=r($t,"CODE",{});var Ld=h(Zo);ur=o(Ld,"Dataset.shuffle()"),Ld.forEach(s),mr=o($t," and "),en=r($t,"CODE",{});var Kd=h(en);fr=o(Kd,"Dataset.select()"),Kd.forEach(s),_r=o($t," functions to create a random sample:"),$t.forEach(s),rl=m(e),j(ea.$$.fragment,e),pl=m(e),j(ta.$$.fragment,e),hl=m(e),Gt=r(e,"P",{});var No=h(Gt);gr=o(No,"Yep, these are certainly movie reviews, and if you\u2019re old enough you may even understand the comment in the last review about owning a VHS version \u{1F61C}! Although we won\u2019t need the labels for language modeling, we can already see that a "),tn=r(No,"CODE",{});var Rd=h(tn);wr=o(Rd,"0"),Rd.forEach(s),kr=o(No," denotes a negative review, while a "),sn=r(No,"CODE",{});var Fd=h(sn);br=o(Fd,"1"),Fd.forEach(s),yr=o(No," corresponds to a positive one."),No.forEach(s),dl=m(e),j(bs.$$.fragment,e),cl=m(e),ys=r(e,"P",{});var Ni=h(ys);vr=o(Ni,"Now that we\u2019ve had a quick look at the data, let\u2019s dive into preparing it for masked language modeling. As we\u2019ll see, there are some additional steps that one needs to take compared to the sequence classification tasks we saw in "),oo=r(Ni,"A",{href:!0});var Nd=h(oo);$r=o(Nd,"Chapter 3"),Nd.forEach(s),jr=o(Ni,". Let\u2019s go!"),Ni.forEach(s),ul=m(e),ls=r(e,"H2",{class:!0});var Wi=h(ls);vs=r(Wi,"A",{id:!0,class:!0,href:!0});var Wd=h(vs);an=r(Wd,"SPAN",{});var Bd=h(an);j(sa.$$.fragment,Bd),Bd.forEach(s),Wd.forEach(s),xr=m(Wi),on=r(Wi,"SPAN",{});var Hd=h(on);Er=o(Hd,"Preprocessing the data"),Hd.forEach(s),Wi.forEach(s),ml=m(e),j(aa.$$.fragment,e),fl=m(e),no=r(e,"P",{});var Id=h(no);Tr=o(Id,"For both auto-regressive and masked language modeling, a common preprocessing step is to concatenate all the examples and then split the whole corpus into chunks of equal size. This is quite different from our usual approach, where we simply tokenize individual examples. Why concatenate everything together? The reason is that individual examples might get truncated if they\u2019re too long, and that would result in losing information that might be useful for the language modeling task!"),Id.forEach(s),_l=m(e),tt=r(e,"P",{});var jt=h(tt);zr=o(jt,"So to get started, we\u2019ll first tokenize our corpus as usual, but "),nn=r(jt,"EM",{});var Gd=h(nn);Ar=o(Gd,"without"),Gd.forEach(s),qr=o(jt," setting the "),ln=r(jt,"CODE",{});var Ud=h(ln);Dr=o(Ud,"truncation=True"),Ud.forEach(s),Mr=o(jt," option in our tokenizer. We\u2019ll also grab the word IDs if they are available ((which they will be if we\u2019re using a fast tokenizer, as described in "),lo=r(jt,"A",{href:!0});var Yd=h(lo);Pr=o(Yd,"Chapter 6"),Yd.forEach(s),Cr=o(jt,"), as we will need them later on to do whole word masking. We\u2019ll wrap this in a simple function, and while we\u2019re at it we\u2019ll remove the "),rn=r(jt,"CODE",{});var Vd=h(rn);Sr=o(Vd,"text"),Vd.forEach(s),Or=o(jt," and "),pn=r(jt,"CODE",{});var Jd=h(pn);Lr=o(Jd,"label"),Jd.forEach(s),Kr=o(jt," columns since we don\u2019t need them any longer:"),jt.forEach(s),gl=m(e),j(oa.$$.fragment,e),wl=m(e),j(na.$$.fragment,e),kl=m(e),kt=r(e,"P",{});var Rs=h(kt);Rr=o(Rs,"Since DistilBERT is a BERT-like model, we can see that the encoded texts consist of the "),hn=r(Rs,"CODE",{});var Qd=h(hn);Fr=o(Qd,"input_ids"),Qd.forEach(s),Nr=o(Rs," and "),dn=r(Rs,"CODE",{});var Xd=h(dn);Wr=o(Xd,"attention_mask"),Xd.forEach(s),Br=o(Rs," that we\u2019ve seen in other chapters, as well as the "),cn=r(Rs,"CODE",{});var Zd=h(cn);Hr=o(Zd,"word_ids"),Zd.forEach(s),Ir=o(Rs," we added."),Rs.forEach(s),bl=m(e),$s=r(e,"P",{});var Bi=h($s);Gr=o(Bi,"Now that we\u2019ve tokenized our movie reviews, the next step is to group them all together and split the result into chunks. But how big should these chunks be? This will ultimately be determined by the amount of GPU memory that you have available, but a good starting point is to see what the model\u2019s maximum context size is. This can be inferred by inspecting the "),un=r(Bi,"CODE",{});var ec=h(un);Ur=o(ec,"model_max_length"),ec.forEach(s),Yr=o(Bi," attribute of the tokenizer:"),Bi.forEach(s),yl=m(e),j(la.$$.fragment,e),vl=m(e),j(ia.$$.fragment,e),$l=m(e),js=r(e,"P",{});var Hi=h(js);Vr=o(Hi,"This value is derived from the "),mn=r(Hi,"EM",{});var tc=h(mn);Jr=o(tc,"tokenizer_config.json"),tc.forEach(s),Qr=o(Hi," file associated with a checkpoint; in this case we can see that the context size is 512 tokens, just like with BERT."),Hi.forEach(s),jl=m(e),j(xs.$$.fragment,e),xl=m(e),io=r(e,"P",{});var sc=h(io);Xr=o(sc,"So, in order to run our experiments on GPUs like those found on Google Colab, we\u2019ll pick something a bit smaller that can fit in memory:"),sc.forEach(s),El=m(e),j(ra.$$.fragment,e),Tl=m(e),j(Es.$$.fragment,e),zl=m(e),ro=r(e,"P",{});var ac=h(ro);Zr=o(ac,"Now comes the fun part. To show how the concatenation works, let\u2019s take a few reviews from our tokenized training set and print out the number of tokens per review:"),ac.forEach(s),Al=m(e),j(pa.$$.fragment,e),ql=m(e),j(ha.$$.fragment,e),Dl=m(e),po=r(e,"P",{});var oc=h(po);ep=o(oc,"We can then concatenate all these examples with a simple dictionary comprehension, as follows:"),oc.forEach(s),Ml=m(e),j(da.$$.fragment,e),Pl=m(e),j(ca.$$.fragment,e),Cl=m(e),Ut=r(e,"P",{});var Wo=h(Ut);tp=o(Wo,"Great, the total length checks out \u2014 so now let\u2019s split the concatenated reviews into chunks of the size given by "),fn=r(Wo,"CODE",{});var nc=h(fn);sp=o(nc,"block_size"),nc.forEach(s),ap=o(Wo,". To do so, we iterate over the features in "),_n=r(Wo,"CODE",{});var lc=h(_n);op=o(lc,"concatenated_examples"),lc.forEach(s),np=o(Wo," and use a list comprehension to create slices of each feature. The result is a dictionary of chunks for each feature:"),Wo.forEach(s),Sl=m(e),j(ua.$$.fragment,e),Ol=m(e),j(ma.$$.fragment,e),Ll=m(e),ho=r(e,"P",{});var ic=h(ho);lp=o(ic,"As you can see in this example, the last chunk will generally be smaller than the maximum chunk size. There are two main strategies for dealing with this:"),ic.forEach(s),Kl=m(e),Ts=r(e,"UL",{});var Ii=h(Ts);fa=r(Ii,"LI",{});var Gi=h(fa);ip=o(Gi,"Drop the last chunk if it\u2019s smaller than "),gn=r(Gi,"CODE",{});var rc=h(gn);rp=o(rc,"chunk_size"),rc.forEach(s),pp=o(Gi,"."),Gi.forEach(s),hp=m(Ii),_a=r(Ii,"LI",{});var Ui=h(_a);dp=o(Ui,"Pad the last chunk until its length equals "),wn=r(Ui,"CODE",{});var pc=h(wn);cp=o(pc,"chunk_size"),pc.forEach(s),up=o(Ui,"."),Ui.forEach(s),Ii.forEach(s),Rl=m(e),co=r(e,"P",{});var hc=h(co);mp=o(hc,"We\u2019ll take the first approach here, so let\u2019s wrap all of the above logic in a single function that we can apply to our tokenized datasets:"),hc.forEach(s),Fl=m(e),j(ga.$$.fragment,e),Nl=m(e),ft=r(e,"P",{});var ts=h(ft);fp=o(ts,"Note that in the last step of "),kn=r(ts,"CODE",{});var dc=h(kn);_p=o(dc,"group_texts()"),dc.forEach(s),gp=o(ts," we create a new "),bn=r(ts,"CODE",{});var cc=h(bn);wp=o(cc,"labels"),cc.forEach(s),kp=o(ts," column which is a copy of the "),yn=r(ts,"CODE",{});var uc=h(yn);bp=o(uc,"input_ids"),uc.forEach(s),yp=o(ts," one. As we\u2019ll see shortly, that\u2019s because in masked language modeling the objective is to predict randomly masked tokens in the input batch, and by creating a "),vn=r(ts,"CODE",{});var mc=h(vn);vp=o(mc,"labels"),mc.forEach(s),$p=o(ts," column we provide the ground truth for our language model to learn from."),ts.forEach(s),Wl=m(e),Yt=r(e,"P",{});var Bo=h(Yt);jp=o(Bo,"Let\u2019s now apply "),$n=r(Bo,"CODE",{});var fc=h($n);xp=o(fc,"group_texts()"),fc.forEach(s),Ep=o(Bo," to our tokenized datasets using our trusty "),jn=r(Bo,"CODE",{});var _c=h(jn);Tp=o(_c,"Dataset.map()"),_c.forEach(s),zp=o(Bo," function:"),Bo.forEach(s),Bl=m(e),j(wa.$$.fragment,e),Hl=m(e),j(ka.$$.fragment,e),Il=m(e),st=r(e,"P",{});var xt=h(st);Ap=o(xt,"You can see that grouping and then chunking the texts has produced many more examples than our original 25,000 for the "),xn=r(xt,"CODE",{});var gc=h(xn);qp=o(gc,"train"),gc.forEach(s),Dp=o(xt," and "),En=r(xt,"CODE",{});var wc=h(En);Mp=o(wc,"test"),wc.forEach(s),Pp=o(xt," splits. That\u2019s because we now have examples involving "),Tn=r(xt,"EM",{});var kc=h(Tn);Cp=o(kc,"contiguous tokens"),kc.forEach(s),Sp=o(xt," that span across multiple examples from the original corpus. You can see this explicitly by looking for the special "),zn=r(xt,"CODE",{});var bc=h(zn);Op=o(bc,"[SEP]"),bc.forEach(s),Lp=o(xt," and "),An=r(xt,"CODE",{});var yc=h(An);Kp=o(yc,"[CLS]"),yc.forEach(s),Rp=o(xt," tokens in one of the chunks:"),xt.forEach(s),Gl=m(e),j(ba.$$.fragment,e),Ul=m(e),j(ya.$$.fragment,e),Yl=m(e),uo=r(e,"P",{});var vc=h(uo);Fp=o(vc,"In this example you can see two overlapping movie reviews, one about a high school movie and the other about homelessness. Let\u2019s also check out what the labels look like for masked language modeling:"),vc.forEach(s),Vl=m(e),j(va.$$.fragment,e),Jl=m(e),j($a.$$.fragment,e),Ql=m(e),bt=r(e,"P",{});var Fs=h(bt);Np=o(Fs,"As expected from our "),qn=r(Fs,"CODE",{});var $c=h(qn);Wp=o($c,"group_texts()"),$c.forEach(s),Bp=o(Fs," function above, this looks identical to the decoded "),Dn=r(Fs,"CODE",{});var jc=h(Dn);Hp=o(jc,"input_ids"),jc.forEach(s),Ip=o(Fs," \u2014 but then how can our model possibly learn anything? We\u2019re missing a key step: inserting "),Mn=r(Fs,"CODE",{});var xc=h(Mn);Gp=o(xc,"[MASK]"),xc.forEach(s),Up=o(Fs," tokens at random positions in the inputs! Let\u2019s see how we can do this on the fly during fine-tuning using a special data collator."),Fs.forEach(s),Xl=m(e),is=r(e,"H2",{class:!0});var Yi=h(is);zs=r(Yi,"A",{id:!0,class:!0,href:!0});var Ec=h(zs);Pn=r(Ec,"SPAN",{});var Tc=h(Pn);j(ja.$$.fragment,Tc),Tc.forEach(s),Ec.forEach(s),Yp=m(Yi),xa=r(Yi,"SPAN",{});var Vi=h(xa);Vp=o(Vi,"Fine-tuning DistilBERT with the "),Cn=r(Vi,"CODE",{});var zc=h(Cn);Jp=o(zc,"Trainer"),zc.forEach(s),Qp=o(Vi," API"),Vi.forEach(s),Yi.forEach(s),Zl=m(e),yt=r(e,"P",{});var Ns=h(yt);Xp=o(Ns,"Fine-tuning a masked language model is almost identical to fine-tuning a sequence classification model, like we did in "),mo=r(Ns,"A",{href:!0});var Ac=h(mo);Zp=o(Ac,"Chapter 3"),Ac.forEach(s),eh=o(Ns,". The only difference is that we need a special data collator that can randomly mask some of the tokens in each batch of texts. Fortunately, \u{1F917} Transformers comes prepared with a dedicated "),Sn=r(Ns,"CODE",{});var qc=h(Sn);th=o(qc,"DataCollatorForLanguageModeling"),qc.forEach(s),sh=o(Ns," for just this task. We just have to pass it the tokenizer and an "),On=r(Ns,"CODE",{});var Dc=h(On);ah=o(Dc,"mlm_probability"),Dc.forEach(s),oh=o(Ns," argument that specifies what fraction of the tokens to mask. We\u2019ll pick 15%, which is the amount used for BERT and a common choice in the literature:"),Ns.forEach(s),ei=m(e),j(Ea.$$.fragment,e),ti=m(e),vt=r(e,"P",{});var Ws=h(vt);nh=o(Ws,"To see how the random masking works, let\u2019s feed a few examples to the data collator. Since it expects a list of "),Ln=r(Ws,"CODE",{});var Mc=h(Ln);lh=o(Mc,"dict"),Mc.forEach(s),ih=o(Ws,"s, where each "),Kn=r(Ws,"CODE",{});var Pc=h(Kn);rh=o(Pc,"dict"),Pc.forEach(s),ph=o(Ws," represents a single chunk of contiguous text, we first iterate over the dataset before feeding the batch to the collator. We remove the "),Rn=r(Ws,"CODE",{});var Cc=h(Rn);hh=o(Cc,'"word_ids"'),Cc.forEach(s),dh=o(Ws," key for this data collator as it does not expect it:"),Ws.forEach(s),si=m(e),j(Ta.$$.fragment,e),ai=m(e),j(za.$$.fragment,e),oi=m(e),Vt=r(e,"P",{});var Ho=h(Vt);ch=o(Ho,"Nice, it worked! We can see that the "),Fn=r(Ho,"CODE",{});var Sc=h(Fn);uh=o(Sc,"[MASK]"),Sc.forEach(s),mh=o(Ho," token has been randomly inserted at various locations in our text. These will be the tokens which our model will have to predict during training \u2014 and the beauty of the data collator is that it will randomize the "),Nn=r(Ho,"CODE",{});var Oc=h(Nn);fh=o(Oc,"[MASK]"),Oc.forEach(s),_h=o(Ho," insertion with every batch!"),Ho.forEach(s),ni=m(e),j(As.$$.fragment,e),li=m(e),it&&it.l(e),fo=m(e),Jt=r(e,"P",{});var Io=h(Jt);gh=o(Io,"When training models for masked language modeling, one technique that can be used is to mask whole words together, not just individual tokens. This approach is called "),Wn=r(Io,"EM",{});var Lc=h(Wn);wh=o(Lc,"whole word masking"),Lc.forEach(s),kh=o(Io,". If we want to use whole word masking, we will need to build a data collator ourselves. A data collator is just a function that takes a list of samples and converts them into a batch, so let\u2019s do this now! We\u2019ll use the word IDs computed earlier to make a map between word indices and the corresponding tokens, then randomly decide which words to mask and apply that mask on the inputs. Note that the labels are all "),Bn=r(Io,"CODE",{});var Kc=h(Bn);bh=o(Kc,"-100"),Kc.forEach(s),yh=o(Io," except for the ones corresponding to mask words."),Io.forEach(s),ii=m(e),Pt.l(e),_o=m(e),go=r(e,"P",{});var Rc=h(go);vh=o(Rc,"Next, we can try it on the same samples as before:"),Rc.forEach(s),ri=m(e),j(Aa.$$.fragment,e),pi=m(e),j(qa.$$.fragment,e),hi=m(e),j(qs.$$.fragment,e),di=m(e),Qt=r(e,"P",{});var Go=h(Qt);$h=o(Go,"Now that we have two data collators, the rest of the fine-tuning steps are standard. Training can take a while on Google Colab if you\u2019re not lucky enough to score a mythical P100 GPU \u{1F62D}, so we\u2019ll first downsample the size of the training set to a few thousand examples. Don\u2019t worry, we\u2019ll still get a pretty decent language model! A quick way to downsample a dataset in \u{1F917} Datasets is via the "),Hn=r(Go,"CODE",{});var Fc=h(Hn);jh=o(Fc,"Dataset.train_test_split()"),Fc.forEach(s),xh=o(Go," function that we saw in "),wo=r(Go,"A",{href:!0});var Nc=h(wo);Eh=o(Nc,"Chapter 5"),Nc.forEach(s),Th=o(Go,":"),Go.forEach(s),ci=m(e),j(Da.$$.fragment,e),ui=m(e),j(Ma.$$.fragment,e),mi=m(e),Xt=r(e,"P",{});var Uo=h(Xt);zh=o(Uo,"This has automatically created new "),In=r(Uo,"CODE",{});var Wc=h(In);Ah=o(Wc,"train"),Wc.forEach(s),qh=o(Uo," and "),Gn=r(Uo,"CODE",{});var Bc=h(Gn);Dh=o(Bc,"test"),Bc.forEach(s),Mh=o(Uo," splits, with the training set size set to 10,000 examples and the validation set to 10% of that \u2014 feel free to increase this if you have a beefy GPU! The next thing we need to do is log in to the Hugging Face Hub. If you\u2019re running this code in a notebook, you can do so with the following utility function:"),Uo.forEach(s),fi=m(e),j(Pa.$$.fragment,e),_i=m(e),ko=r(e,"P",{});var Hc=h(ko);Ph=o(Hc,"which will display a widget where you can enter your credentials. Alternatively, you can run:"),Hc.forEach(s),gi=m(e),j(Ca.$$.fragment,e),wi=m(e),bo=r(e,"P",{});var Ic=h(bo);Ch=o(Ic,"in your favorite terminal and log in there."),Ic.forEach(s),ki=m(e),St.l(e),yo=m(e),rs=r(e,"H3",{class:!0});var Ji=h(rs);Ds=r(Ji,"A",{id:!0,class:!0,href:!0});var Gc=h(Ds);Un=r(Gc,"SPAN",{});var Uc=h(Un);j(Sa.$$.fragment,Uc),Uc.forEach(s),Gc.forEach(s),Sh=m(Ji),Yn=r(Ji,"SPAN",{});var Yc=h(Yn);Oh=o(Yc,"Perplexity for language models"),Yc.forEach(s),Ji.forEach(s),bi=m(e),j(Oa.$$.fragment,e),yi=m(e),vo=r(e,"P",{});var Vc=h(vo);Lh=o(Vc,"Unlike other tasks like text classification or question answering where we\u2019re given a labeled corpus to train on, with language modeling we don\u2019t have any explicit labels. So how do we determine what makes a good language model? Like with the autocorrect feature in your phone, a good language model is one that assigns high probabilities to sentences that are grammatically correct, and low probabilities to nonsense sentences. To give you a better idea of what this looks like, you can find whole sets of \u201Cautocorrect fails\u201D online, where the model in a person\u2019s phone has produced some rather funny (and often inappropriate) completions!"),Vc.forEach(s),vi=m(e),Lt.l(e),$o=m(e),j(La.$$.fragment,e),$i=m(e),jo=r(e,"P",{});var Jc=h(jo);Kh=o(Jc,"A lower perplexity score means a better language model, and we can see here that our starting model has a somewhat large value. Let\u2019s see if we can lower it by fine-tuning! To do that, we first run the training loop:"),Jc.forEach(s),ji=m(e),Rt.l(e),xo=m(e),Eo=r(e,"P",{});var Qc=h(Eo);Rh=o(Qc,"and then compute the resulting perplexity on the test set as before:"),Qc.forEach(s),xi=m(e),Nt.l(e),To=m(e),j(Ka.$$.fragment,e),Ei=m(e),zo=r(e,"P",{});var Xc=h(zo);Fh=o(Xc,"Nice \u2014 this is quite a reduction in perplexity, which tells us the model has learned something about the domain of movie reviews!"),Xc.forEach(s),Ti=m(e),Oe&&Oe.l(e),Ao=m(e),j(Ms.$$.fragment,e),zi=m(e),Le&&Le.l(e),qo=m(e),ps=r(e,"H2",{class:!0});var Qi=h(ps);Ps=r(Qi,"A",{id:!0,class:!0,href:!0});var Zc=h(Ps);Vn=r(Zc,"SPAN",{});var eu=h(Vn);j(Ra.$$.fragment,eu),eu.forEach(s),Zc.forEach(s),Nh=m(Qi),Jn=r(Qi,"SPAN",{});var tu=h(Jn);Wh=o(tu,"Using our fine-tuned model"),tu.forEach(s),Qi.forEach(s),Ai=m(e),Zt=r(e,"P",{});var Yo=h(Zt);Bh=o(Yo,"You can interact with your fine-tuned model either by using its widget on the Hub or locally with the "),Qn=r(Yo,"CODE",{});var su=h(Qn);Hh=o(su,"pipeline"),su.forEach(s),Ih=o(Yo," from \u{1F917} Transformers. Let\u2019s use the latter to download our model using the "),Xn=r(Yo,"CODE",{});var au=h(Xn);Gh=o(au,"fill-mask"),au.forEach(s),Uh=o(Yo," pipeline:"),Yo.forEach(s),qi=m(e),j(Fa.$$.fragment,e),Di=m(e),Do=r(e,"P",{});var ou=h(Do);Yh=o(ou,"We can then feed the pipeline our sample text of \u201CThis is a great [MASK]\u201D and see what the top 5 predictions are:"),ou.forEach(s),Mi=m(e),j(Na.$$.fragment,e),Pi=m(e),j(Wa.$$.fragment,e),Ci=m(e),Mo=r(e,"P",{});var nu=h(Mo);Vh=o(nu,"Neat \u2014 our model has clearly adapted its weights to predict words that are more strongly associated with movies!"),nu.forEach(s),Si=m(e),j(Ba.$$.fragment,e),Oi=m(e),Cs=r(e,"P",{});var Xi=h(Cs);Jh=o(Xi,"This wraps up our first experiment with training a language model. In "),Po=r(Xi,"A",{href:!0});var lu=h(Po);Qh=o(lu,"section 6"),lu.forEach(s),Xh=o(Xi," you\u2019ll learn how to train an auto-regressive model like GPT-2 from scratch; head over there if you\u2019d like to see how you can pretrain your very own Transformer model!"),Xi.forEach(s),Li=m(e),j(Ss.$$.fragment,e),this.h()},h(){T(l,"name","hf:doc:metadata"),T(l,"content",JSON.stringify(Wu)),T(k,"id","fine-tuning-a-masked-language-model"),T(k,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),T(k,"href","#fine-tuning-a-masked-language-model"),T(z,"class","relative group"),T(le,"href","https://arxiv.org/abs/1801.06146"),T(le,"rel","nofollow"),T(te,"class","block dark:hidden"),Zi(te.src,ye="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter7/ulmfit.svg")||T(te,"src",ye),T(te,"alt","ULMFiT."),T(Q,"class","hidden dark:block"),Zi(Q.src,De="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter7/ulmfit-dark.svg")||T(Q,"src",De),T(Q,"alt","ULMFiT."),T(ce,"class","flex justify-center"),T(ge,"href","https://huggingface.co/huggingface-course/distilbert-base-uncased-finetuned-imdb?text=This+is+a+great+%5BMASK%5D."),T(ge,"rel","nofollow"),Zi(pe.src,Qe="https://course-demos-distilbert-base-uncased-finetune-7400b54.hf.space")||T(pe,"src",Qe),T(pe,"frameborder","0"),T(pe,"height","300"),T(pe,"title","Gradio app"),T(pe,"class","block dark:hidden container p-0 flex-grow space-iframe"),T(pe,"allow","accelerometer; ambient-light-sensor; autoplay; battery; camera; document-domain; encrypted-media; fullscreen; geolocation; gyroscope; layout-animations; legacy-image-formats; magnetometer; microphone; midi; oversized-images; payment; picture-in-picture; publickey-credentials-get; sync-xhr; usb; vr ; wake-lock; xr-spatial-tracking"),T(pe,"sandbox","allow-forms allow-modals allow-popups allow-popups-to-escape-sandbox allow-same-origin allow-scripts allow-downloads"),T(me,"id","picking-a-pretrained-model-for-masked-language-modeling"),T(me,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),T(me,"href","#picking-a-pretrained-model-for-masked-language-modeling"),T(se,"class","relative group"),T(fe,"href","https://huggingface.co/models?pipeline_tag=fill-mask&sort=downloads"),T(fe,"rel","nofollow"),Zi(Ee.src,S="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter7/mlm-models.png")||T(Ee,"src",S),T(Ee,"alt","Hub models."),T(Ee,"width","80%"),T(Ne,"class","flex justify-center"),T(Pe,"href","https://huggingface.co/distilbert-base-uncased"),T(Pe,"rel","nofollow"),T(Te,"href","https://en.wikipedia.org/wiki/Knowledge_distillation"),T(Te,"rel","nofollow"),T(ot,"href","https://www.oreilly.com/library/view/natural-language-processing/9781098136789/"),T(ot,"rel","nofollow"),T(ct,"href","https://huggingface.co/datasets/wikipedia"),T(ct,"rel","nofollow"),T(c,"href","https://huggingface.co/datasets/bookcorpus"),T(c,"rel","nofollow"),T(Dt,"id","the-dataset"),T(Dt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),T(Dt,"href","#the-dataset"),T(mt,"class","relative group"),T(Qs,"href","https://huggingface.co/datasets/imdb"),T(Qs,"rel","nofollow"),T(oo,"href","/course/chapter3"),T(vs,"id","preprocessing-the-data"),T(vs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),T(vs,"href","#preprocessing-the-data"),T(ls,"class","relative group"),T(lo,"href","/course/chapter6/3"),T(zs,"id","fine-tuning-distilbert-with-the-trainer-api"),T(zs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),T(zs,"href","#fine-tuning-distilbert-with-the-trainer-api"),T(is,"class","relative group"),T(mo,"href","/course/chapter3"),T(wo,"href","/course/chapter5"),T(Ds,"id","perplexity-for-language-models"),T(Ds,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),T(Ds,"href","#perplexity-for-language-models"),T(rs,"class","relative group"),T(Ps,"id","using-our-fine-tuned-model"),T(Ps,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),T(Ps,"href","#using-our-fine-tuned-model"),T(ps,"class","relative group"),T(Po,"href","/course/chapter7/section6")},m(e,n){t(document.head,l),p(e,f,n),x(d,e,n),p(e,b,n),p(e,z,n),t(z,k),t(k,y),x(q,y,null),t(z,g),t(z,D),t(D,F),p(e,O,n),Ha[M].m(e,n),p(e,P,n),p(e,W,n),t(W,B),p(e,J,n),p(e,v,n),t(v,R),p(e,U,n),p(e,Y,n),t(Y,X),t(Y,ue),t(ue,Ae),t(Y,ne),t(Y,le),t(le,V),t(Y,_e),p(e,H,n),p(e,ce,n),t(ce,te),t(ce,qe),t(ce,Q),p(e,Ke,n),p(e,ie,n),t(ie,Be),t(ie,ge),t(ge,Me),t(ie,Je),p(e,re,n),p(e,pe,n),p(e,je,n),p(e,Re,n),t(Re,Xe),p(e,we,n),x(Z,e,n),p(e,xe,n),x(ee,e,n),p(e,He,n),p(e,se,n),t(se,me),t(me,ke),x(ve,ke,null),t(se,Fe),t(se,$e),t($e,Ie),p(e,C,n),p(e,I,n),t(I,Ge),t(I,fe),t(fe,rt),t(I,oe),p(e,at,n),p(e,Ne,n),t(Ne,Ee),p(e,G,n),p(e,he,n),t(he,de),t(he,Pe),t(Pe,Ue),t(he,pt),t(he,Te),t(Te,Ce),t(Ce,ht),t(he,ae),t(he,ot),t(ot,Ye),t(Ye,Et),t(he,nt),p(e,Tt,n),Ia[ze].m(e,n),p(e,_t,n),p(e,dt,n),t(dt,We),p(e,gt,n),x(Ve,e,n),p(e,ds,n),p(e,Se,n),t(Se,zt),t(Se,Wt),t(Wt,At),t(Se,cs),t(Se,ct),t(ct,Is),t(Se,us),t(Se,c),t(c,L),t(Se,ms),t(Se,fs),t(fs,eo),t(Se,to),p(e,qt,n),x(Bt,e,n),p(e,Gs,n),p(e,lt,n),t(lt,so),p(e,Us,n),Ga[ut].m(e,n),p(e,os,n),x(wt,e,n),p(e,Ys,n),p(e,ns,n),t(ns,_s),p(e,Vs,n),p(e,mt,n),t(mt,Dt),t(Dt,gs),x(Ht,gs,null),t(mt,ws),t(mt,ks),t(ks,ao),p(e,Js,n),p(e,It,n),t(It,tr),t(It,Qs),t(Qs,sr),t(It,ar),t(It,Vo),t(Vo,or),t(It,nr),p(e,nl,n),x(Xs,e,n),p(e,ll,n),x(Zs,e,n),p(e,il,n),p(e,et,n),t(et,lr),t(et,Jo),t(Jo,ir),t(et,rr),t(et,Qo),t(Qo,pr),t(et,hr),t(et,Xo),t(Xo,dr),t(et,cr),t(et,Zo),t(Zo,ur),t(et,mr),t(et,en),t(en,fr),t(et,_r),p(e,rl,n),x(ea,e,n),p(e,pl,n),x(ta,e,n),p(e,hl,n),p(e,Gt,n),t(Gt,gr),t(Gt,tn),t(tn,wr),t(Gt,kr),t(Gt,sn),t(sn,br),t(Gt,yr),p(e,dl,n),x(bs,e,n),p(e,cl,n),p(e,ys,n),t(ys,vr),t(ys,oo),t(oo,$r),t(ys,jr),p(e,ul,n),p(e,ls,n),t(ls,vs),t(vs,an),x(sa,an,null),t(ls,xr),t(ls,on),t(on,Er),p(e,ml,n),x(aa,e,n),p(e,fl,n),p(e,no,n),t(no,Tr),p(e,_l,n),p(e,tt,n),t(tt,zr),t(tt,nn),t(nn,Ar),t(tt,qr),t(tt,ln),t(ln,Dr),t(tt,Mr),t(tt,lo),t(lo,Pr),t(tt,Cr),t(tt,rn),t(rn,Sr),t(tt,Or),t(tt,pn),t(pn,Lr),t(tt,Kr),p(e,gl,n),x(oa,e,n),p(e,wl,n),x(na,e,n),p(e,kl,n),p(e,kt,n),t(kt,Rr),t(kt,hn),t(hn,Fr),t(kt,Nr),t(kt,dn),t(dn,Wr),t(kt,Br),t(kt,cn),t(cn,Hr),t(kt,Ir),p(e,bl,n),p(e,$s,n),t($s,Gr),t($s,un),t(un,Ur),t($s,Yr),p(e,yl,n),x(la,e,n),p(e,vl,n),x(ia,e,n),p(e,$l,n),p(e,js,n),t(js,Vr),t(js,mn),t(mn,Jr),t(js,Qr),p(e,jl,n),x(xs,e,n),p(e,xl,n),p(e,io,n),t(io,Xr),p(e,El,n),x(ra,e,n),p(e,Tl,n),x(Es,e,n),p(e,zl,n),p(e,ro,n),t(ro,Zr),p(e,Al,n),x(pa,e,n),p(e,ql,n),x(ha,e,n),p(e,Dl,n),p(e,po,n),t(po,ep),p(e,Ml,n),x(da,e,n),p(e,Pl,n),x(ca,e,n),p(e,Cl,n),p(e,Ut,n),t(Ut,tp),t(Ut,fn),t(fn,sp),t(Ut,ap),t(Ut,_n),t(_n,op),t(Ut,np),p(e,Sl,n),x(ua,e,n),p(e,Ol,n),x(ma,e,n),p(e,Ll,n),p(e,ho,n),t(ho,lp),p(e,Kl,n),p(e,Ts,n),t(Ts,fa),t(fa,ip),t(fa,gn),t(gn,rp),t(fa,pp),t(Ts,hp),t(Ts,_a),t(_a,dp),t(_a,wn),t(wn,cp),t(_a,up),p(e,Rl,n),p(e,co,n),t(co,mp),p(e,Fl,n),x(ga,e,n),p(e,Nl,n),p(e,ft,n),t(ft,fp),t(ft,kn),t(kn,_p),t(ft,gp),t(ft,bn),t(bn,wp),t(ft,kp),t(ft,yn),t(yn,bp),t(ft,yp),t(ft,vn),t(vn,vp),t(ft,$p),p(e,Wl,n),p(e,Yt,n),t(Yt,jp),t(Yt,$n),t($n,xp),t(Yt,Ep),t(Yt,jn),t(jn,Tp),t(Yt,zp),p(e,Bl,n),x(wa,e,n),p(e,Hl,n),x(ka,e,n),p(e,Il,n),p(e,st,n),t(st,Ap),t(st,xn),t(xn,qp),t(st,Dp),t(st,En),t(En,Mp),t(st,Pp),t(st,Tn),t(Tn,Cp),t(st,Sp),t(st,zn),t(zn,Op),t(st,Lp),t(st,An),t(An,Kp),t(st,Rp),p(e,Gl,n),x(ba,e,n),p(e,Ul,n),x(ya,e,n),p(e,Yl,n),p(e,uo,n),t(uo,Fp),p(e,Vl,n),x(va,e,n),p(e,Jl,n),x($a,e,n),p(e,Ql,n),p(e,bt,n),t(bt,Np),t(bt,qn),t(qn,Wp),t(bt,Bp),t(bt,Dn),t(Dn,Hp),t(bt,Ip),t(bt,Mn),t(Mn,Gp),t(bt,Up),p(e,Xl,n),p(e,is,n),t(is,zs),t(zs,Pn),x(ja,Pn,null),t(is,Yp),t(is,xa),t(xa,Vp),t(xa,Cn),t(Cn,Jp),t(xa,Qp),p(e,Zl,n),p(e,yt,n),t(yt,Xp),t(yt,mo),t(mo,Zp),t(yt,eh),t(yt,Sn),t(Sn,th),t(yt,sh),t(yt,On),t(On,ah),t(yt,oh),p(e,ei,n),x(Ea,e,n),p(e,ti,n),p(e,vt,n),t(vt,nh),t(vt,Ln),t(Ln,lh),t(vt,ih),t(vt,Kn),t(Kn,rh),t(vt,ph),t(vt,Rn),t(Rn,hh),t(vt,dh),p(e,si,n),x(Ta,e,n),p(e,ai,n),x(za,e,n),p(e,oi,n),p(e,Vt,n),t(Vt,ch),t(Vt,Fn),t(Fn,uh),t(Vt,mh),t(Vt,Nn),t(Nn,fh),t(Vt,_h),p(e,ni,n),x(As,e,n),p(e,li,n),it&&it.m(e,n),p(e,fo,n),p(e,Jt,n),t(Jt,gh),t(Jt,Wn),t(Wn,wh),t(Jt,kh),t(Jt,Bn),t(Bn,bh),t(Jt,yh),p(e,ii,n),Ua[Mt].m(e,n),p(e,_o,n),p(e,go,n),t(go,vh),p(e,ri,n),x(Aa,e,n),p(e,pi,n),x(qa,e,n),p(e,hi,n),x(qs,e,n),p(e,di,n),p(e,Qt,n),t(Qt,$h),t(Qt,Hn),t(Hn,jh),t(Qt,xh),t(Qt,wo),t(wo,Eh),t(Qt,Th),p(e,ci,n),x(Da,e,n),p(e,ui,n),x(Ma,e,n),p(e,mi,n),p(e,Xt,n),t(Xt,zh),t(Xt,In),t(In,Ah),t(Xt,qh),t(Xt,Gn),t(Gn,Dh),t(Xt,Mh),p(e,fi,n),x(Pa,e,n),p(e,_i,n),p(e,ko,n),t(ko,Ph),p(e,gi,n),x(Ca,e,n),p(e,wi,n),p(e,bo,n),t(bo,Ch),p(e,ki,n),Ya[Ct].m(e,n),p(e,yo,n),p(e,rs,n),t(rs,Ds),t(Ds,Un),x(Sa,Un,null),t(rs,Sh),t(rs,Yn),t(Yn,Oh),p(e,bi,n),x(Oa,e,n),p(e,yi,n),p(e,vo,n),t(vo,Lh),p(e,vi,n),Va[Ot].m(e,n),p(e,$o,n),x(La,e,n),p(e,$i,n),p(e,jo,n),t(jo,Kh),p(e,ji,n),Ja[Kt].m(e,n),p(e,xo,n),p(e,Eo,n),t(Eo,Rh),p(e,xi,n),Qa[Ft].m(e,n),p(e,To,n),x(Ka,e,n),p(e,Ei,n),p(e,zo,n),t(zo,Fh),p(e,Ti,n),Oe&&Oe.m(e,n),p(e,Ao,n),x(Ms,e,n),p(e,zi,n),Le&&Le.m(e,n),p(e,qo,n),p(e,ps,n),t(ps,Ps),t(Ps,Vn),x(Ra,Vn,null),t(ps,Nh),t(ps,Jn),t(Jn,Wh),p(e,Ai,n),p(e,Zt,n),t(Zt,Bh),t(Zt,Qn),t(Qn,Hh),t(Zt,Ih),t(Zt,Xn),t(Xn,Gh),t(Zt,Uh),p(e,qi,n),x(Fa,e,n),p(e,Di,n),p(e,Do,n),t(Do,Yh),p(e,Mi,n),x(Na,e,n),p(e,Pi,n),x(Wa,e,n),p(e,Ci,n),p(e,Mo,n),t(Mo,Vh),p(e,Si,n),x(Ba,e,n),p(e,Oi,n),p(e,Cs,n),t(Cs,Jh),t(Cs,Po),t(Po,Qh),t(Cs,Xh),p(e,Li,n),x(Ss,e,n),Ki=!0},p(e,[n]){const Xa={};n&1&&(Xa.fw=e[0]),d.$set(Xa);let Co=M;M=ed(e),M!==Co&&(as(),w(Ha[Co],1,1,()=>{Ha[Co]=null}),ss(),N=Ha[M],N||(N=Ha[M]=Zh[M](e),N.c()),_(N,1),N.m(P.parentNode,P));const Zn={};n&2&&(Zn.$$scope={dirty:n,ctx:e}),ee.$set(Zn);let So=ze;ze=sd(e),ze!==So&&(as(),w(Ia[So],1,1,()=>{Ia[So]=null}),ss(),be=Ia[ze],be||(be=Ia[ze]=td[ze](e),be.c()),_(be,1),be.m(_t.parentNode,_t));let Oo=ut;ut=od(e),ut!==Oo&&(as(),w(Ga[Oo],1,1,()=>{Ga[Oo]=null}),ss(),Ze=Ga[ut],Ze||(Ze=Ga[ut]=ad[ut](e),Ze.c()),_(Ze,1),Ze.m(os.parentNode,os));const el={};n&2&&(el.$$scope={dirty:n,ctx:e}),bs.$set(el);const hs={};n&2&&(hs.$$scope={dirty:n,ctx:e}),xs.$set(hs);const tl={};n&2&&(tl.$$scope={dirty:n,ctx:e}),Es.$set(tl);const sl={};n&2&&(sl.$$scope={dirty:n,ctx:e}),As.$set(sl),e[0]==="pt"?it||(it=iu(),it.c(),it.m(fo.parentNode,fo)):it&&(it.d(1),it=null);let Os=Mt;Mt=ld(e),Mt!==Os&&(as(),w(Ua[Os],1,1,()=>{Ua[Os]=null}),ss(),Pt=Ua[Mt],Pt||(Pt=Ua[Mt]=nd[Mt](e),Pt.c()),_(Pt,1),Pt.m(_o.parentNode,_o));const Za={};n&2&&(Za.$$scope={dirty:n,ctx:e}),qs.$set(Za);let Lo=Ct;Ct=rd(e),Ct!==Lo&&(as(),w(Ya[Lo],1,1,()=>{Ya[Lo]=null}),ss(),St=Ya[Ct],St||(St=Ya[Ct]=id[Ct](e),St.c()),_(St,1),St.m(yo.parentNode,yo));let Ko=Ot;Ot=hd(e),Ot!==Ko&&(as(),w(Va[Ko],1,1,()=>{Va[Ko]=null}),ss(),Lt=Va[Ot],Lt||(Lt=Va[Ot]=pd[Ot](e),Lt.c()),_(Lt,1),Lt.m($o.parentNode,$o));let Ls=Kt;Kt=cd(e),Kt!==Ls&&(as(),w(Ja[Ls],1,1,()=>{Ja[Ls]=null}),ss(),Rt=Ja[Kt],Rt||(Rt=Ja[Kt]=dd[Kt](e),Rt.c()),_(Rt,1),Rt.m(xo.parentNode,xo));let Ro=Ft;Ft=md(e),Ft!==Ro&&(as(),w(Qa[Ro],1,1,()=>{Qa[Ro]=null}),ss(),Nt=Qa[Ft],Nt||(Nt=Qa[Ft]=ud[Ft](e),Nt.c()),_(Nt,1),Nt.m(To.parentNode,To)),e[0]==="pt"?Oe?n&1&&_(Oe,1):(Oe=ru(),Oe.c(),_(Oe,1),Oe.m(Ao.parentNode,Ao)):Oe&&(as(),w(Oe,1,1,()=>{Oe=null}),ss());const al={};n&2&&(al.$$scope={dirty:n,ctx:e}),Ms.$set(al),e[0]==="pt"?Le?n&1&&_(Le,1):(Le=pu(),Le.c(),_(Le,1),Le.m(qo.parentNode,qo)):Le&&(as(),w(Le,1,1,()=>{Le=null}),ss());const ol={};n&2&&(ol.$$scope={dirty:n,ctx:e}),Ss.$set(ol)},i(e){Ki||(_(d.$$.fragment,e),_(q.$$.fragment,e),_(N),_(Z.$$.fragment,e),_(ee.$$.fragment,e),_(ve.$$.fragment,e),_(be),_(Ve.$$.fragment,e),_(Bt.$$.fragment,e),_(Ze),_(wt.$$.fragment,e),_(Ht.$$.fragment,e),_(Xs.$$.fragment,e),_(Zs.$$.fragment,e),_(ea.$$.fragment,e),_(ta.$$.fragment,e),_(bs.$$.fragment,e),_(sa.$$.fragment,e),_(aa.$$.fragment,e),_(oa.$$.fragment,e),_(na.$$.fragment,e),_(la.$$.fragment,e),_(ia.$$.fragment,e),_(xs.$$.fragment,e),_(ra.$$.fragment,e),_(Es.$$.fragment,e),_(pa.$$.fragment,e),_(ha.$$.fragment,e),_(da.$$.fragment,e),_(ca.$$.fragment,e),_(ua.$$.fragment,e),_(ma.$$.fragment,e),_(ga.$$.fragment,e),_(wa.$$.fragment,e),_(ka.$$.fragment,e),_(ba.$$.fragment,e),_(ya.$$.fragment,e),_(va.$$.fragment,e),_($a.$$.fragment,e),_(ja.$$.fragment,e),_(Ea.$$.fragment,e),_(Ta.$$.fragment,e),_(za.$$.fragment,e),_(As.$$.fragment,e),_(Pt),_(Aa.$$.fragment,e),_(qa.$$.fragment,e),_(qs.$$.fragment,e),_(Da.$$.fragment,e),_(Ma.$$.fragment,e),_(Pa.$$.fragment,e),_(Ca.$$.fragment,e),_(St),_(Sa.$$.fragment,e),_(Oa.$$.fragment,e),_(Lt),_(La.$$.fragment,e),_(Rt),_(Nt),_(Ka.$$.fragment,e),_(Oe),_(Ms.$$.fragment,e),_(Le),_(Ra.$$.fragment,e),_(Fa.$$.fragment,e),_(Na.$$.fragment,e),_(Wa.$$.fragment,e),_(Ba.$$.fragment,e),_(Ss.$$.fragment,e),Ki=!0)},o(e){w(d.$$.fragment,e),w(q.$$.fragment,e),w(N),w(Z.$$.fragment,e),w(ee.$$.fragment,e),w(ve.$$.fragment,e),w(be),w(Ve.$$.fragment,e),w(Bt.$$.fragment,e),w(Ze),w(wt.$$.fragment,e),w(Ht.$$.fragment,e),w(Xs.$$.fragment,e),w(Zs.$$.fragment,e),w(ea.$$.fragment,e),w(ta.$$.fragment,e),w(bs.$$.fragment,e),w(sa.$$.fragment,e),w(aa.$$.fragment,e),w(oa.$$.fragment,e),w(na.$$.fragment,e),w(la.$$.fragment,e),w(ia.$$.fragment,e),w(xs.$$.fragment,e),w(ra.$$.fragment,e),w(Es.$$.fragment,e),w(pa.$$.fragment,e),w(ha.$$.fragment,e),w(da.$$.fragment,e),w(ca.$$.fragment,e),w(ua.$$.fragment,e),w(ma.$$.fragment,e),w(ga.$$.fragment,e),w(wa.$$.fragment,e),w(ka.$$.fragment,e),w(ba.$$.fragment,e),w(ya.$$.fragment,e),w(va.$$.fragment,e),w($a.$$.fragment,e),w(ja.$$.fragment,e),w(Ea.$$.fragment,e),w(Ta.$$.fragment,e),w(za.$$.fragment,e),w(As.$$.fragment,e),w(Pt),w(Aa.$$.fragment,e),w(qa.$$.fragment,e),w(qs.$$.fragment,e),w(Da.$$.fragment,e),w(Ma.$$.fragment,e),w(Pa.$$.fragment,e),w(Ca.$$.fragment,e),w(St),w(Sa.$$.fragment,e),w(Oa.$$.fragment,e),w(Lt),w(La.$$.fragment,e),w(Rt),w(Nt),w(Ka.$$.fragment,e),w(Oe),w(Ms.$$.fragment,e),w(Le),w(Ra.$$.fragment,e),w(Fa.$$.fragment,e),w(Na.$$.fragment,e),w(Wa.$$.fragment,e),w(Ba.$$.fragment,e),w(Ss.$$.fragment,e),Ki=!1},d(e){s(l),e&&s(f),E(d,e),e&&s(b),e&&s(z),E(q),e&&s(O),Ha[M].d(e),e&&s(P),e&&s(W),e&&s(J),e&&s(v),e&&s(U),e&&s(Y),e&&s(H),e&&s(ce),e&&s(Ke),e&&s(ie),e&&s(re),e&&s(pe),e&&s(je),e&&s(Re),e&&s(we),E(Z,e),e&&s(xe),E(ee,e),e&&s(He),e&&s(se),E(ve),e&&s(C),e&&s(I),e&&s(at),e&&s(Ne),e&&s(G),e&&s(he),e&&s(Tt),Ia[ze].d(e),e&&s(_t),e&&s(dt),e&&s(gt),E(Ve,e),e&&s(ds),e&&s(Se),e&&s(qt),E(Bt,e),e&&s(Gs),e&&s(lt),e&&s(Us),Ga[ut].d(e),e&&s(os),E(wt,e),e&&s(Ys),e&&s(ns),e&&s(Vs),e&&s(mt),E(Ht),e&&s(Js),e&&s(It),e&&s(nl),E(Xs,e),e&&s(ll),E(Zs,e),e&&s(il),e&&s(et),e&&s(rl),E(ea,e),e&&s(pl),E(ta,e),e&&s(hl),e&&s(Gt),e&&s(dl),E(bs,e),e&&s(cl),e&&s(ys),e&&s(ul),e&&s(ls),E(sa),e&&s(ml),E(aa,e),e&&s(fl),e&&s(no),e&&s(_l),e&&s(tt),e&&s(gl),E(oa,e),e&&s(wl),E(na,e),e&&s(kl),e&&s(kt),e&&s(bl),e&&s($s),e&&s(yl),E(la,e),e&&s(vl),E(ia,e),e&&s($l),e&&s(js),e&&s(jl),E(xs,e),e&&s(xl),e&&s(io),e&&s(El),E(ra,e),e&&s(Tl),E(Es,e),e&&s(zl),e&&s(ro),e&&s(Al),E(pa,e),e&&s(ql),E(ha,e),e&&s(Dl),e&&s(po),e&&s(Ml),E(da,e),e&&s(Pl),E(ca,e),e&&s(Cl),e&&s(Ut),e&&s(Sl),E(ua,e),e&&s(Ol),E(ma,e),e&&s(Ll),e&&s(ho),e&&s(Kl),e&&s(Ts),e&&s(Rl),e&&s(co),e&&s(Fl),E(ga,e),e&&s(Nl),e&&s(ft),e&&s(Wl),e&&s(Yt),e&&s(Bl),E(wa,e),e&&s(Hl),E(ka,e),e&&s(Il),e&&s(st),e&&s(Gl),E(ba,e),e&&s(Ul),E(ya,e),e&&s(Yl),e&&s(uo),e&&s(Vl),E(va,e),e&&s(Jl),E($a,e),e&&s(Ql),e&&s(bt),e&&s(Xl),e&&s(is),E(ja),e&&s(Zl),e&&s(yt),e&&s(ei),E(Ea,e),e&&s(ti),e&&s(vt),e&&s(si),E(Ta,e),e&&s(ai),E(za,e),e&&s(oi),e&&s(Vt),e&&s(ni),E(As,e),e&&s(li),it&&it.d(e),e&&s(fo),e&&s(Jt),e&&s(ii),Ua[Mt].d(e),e&&s(_o),e&&s(go),e&&s(ri),E(Aa,e),e&&s(pi),E(qa,e),e&&s(hi),E(qs,e),e&&s(di),e&&s(Qt),e&&s(ci),E(Da,e),e&&s(ui),E(Ma,e),e&&s(mi),e&&s(Xt),e&&s(fi),E(Pa,e),e&&s(_i),e&&s(ko),e&&s(gi),E(Ca,e),e&&s(wi),e&&s(bo),e&&s(ki),Ya[Ct].d(e),e&&s(yo),e&&s(rs),E(Sa),e&&s(bi),E(Oa,e),e&&s(yi),e&&s(vo),e&&s(vi),Va[Ot].d(e),e&&s($o),E(La,e),e&&s($i),e&&s(jo),e&&s(ji),Ja[Kt].d(e),e&&s(xo),e&&s(Eo),e&&s(xi),Qa[Ft].d(e),e&&s(To),E(Ka,e),e&&s(Ei),e&&s(zo),e&&s(Ti),Oe&&Oe.d(e),e&&s(Ao),E(Ms,e),e&&s(zi),Le&&Le.d(e),e&&s(qo),e&&s(ps),E(Ra),e&&s(Ai),e&&s(Zt),e&&s(qi),E(Fa,e),e&&s(Di),e&&s(Do),e&&s(Mi),E(Na,e),e&&s(Pi),E(Wa,e),e&&s(Ci),e&&s(Mo),e&&s(Si),E(Ba,e),e&&s(Oi),e&&s(Cs),e&&s(Li),E(Ss,e)}}}const Wu={local:"fine-tuning-a-masked-language-model",sections:[{local:"picking-a-pretrained-model-for-masked-language-modeling",title:"Picking a pretrained model for masked language modeling"},{local:"the-dataset",title:"The dataset"},{local:"preprocessing-the-data",title:"Preprocessing the data"},{local:"fine-tuning-distilbert-with-the-trainer-api",sections:[{local:"perplexity-for-language-models",title:"Perplexity for language models"}],title:"Fine-tuning DistilBERT with the `Trainer` API"},{local:"fine-tuning-distilbert-with-accelerate",title:"Fine-tuning DistilBERT with \u{1F917} Accelerate"},{local:"using-our-fine-tuned-model",title:"Using our fine-tuned model"}],title:"Fine-tuning a masked language model"};function Bu(K,l,f){let d="pt";return fu(()=>{const b=new URLSearchParams(window.location.search);f(0,d=b.get("fw")||"pt")}),[d]}class Qu extends du{constructor(l){super();cu(this,l,Bu,Nu,uu,{})}}export{Qu as default,Wu as metadata};
