---
license: mit
datasets:
- jhu-clsp/mmbert-decay
- jhu-clsp/mmbert-midtraining
- jhu-clsp/mmbert-pretrain-p1-fineweb2-langs
- jhu-clsp/mmbert-pretrain-p2-fineweb2-remaining
- jhu-clsp/mmbert-pretrain-p3-others
pipeline_tag: fill-mask
library_name: transformers
language:
 - aai
 - aak
 - aau
 - aaz
 - aba
 - abi
 - abk
 - abn
 - abq
 - abs
 - abt
 - abx
 - aby
 - abz
 - aca
 - acd
 - ace
 - acf
 - ach
 - acm
 - acn
 - acr
 - acu
 - ada
 - ade
 - adh
 - adi
 - adj
 - adl
 - ady
 - adz
 - aeb
 - aer
 - aeu
 - aey
 - afr
 - agd
 - agg
 - agm
 - agn
 - agr
 - agt
 - agu
 - agw
 - agx
 - aha
 - ahk
 - aia
 - aii
 - aim
 - ain
 - ajg
 - aji
 - ajz
 - akb
 - ake
 - akh
 - akp
 - alj
 - aln
 - alp
 - alq
 - als
 - alt
 - aly
 - alz
 - ame
 - amf
 - amh
 - ami
 - amk
 - amm
 - amn
 - amp
 - amr
 - amu
 - amx
 - ang
 - anm
 - ann
 - anp
 - anv
 - any
 - aoi
 - aoj
 - aom
 - aoz
 - apb
 - apc
 - ape
 - apn
 - apr
 - apt
 - apu
 - apw
 - apy
 - apz
 - arb
 - are
 - arg
 - arl
 - arn
 - arp
 - arq
 - ars
 - ary
 - arz
 - asg
 - asm
 - aso
 - ast
 - ata
 - atb
 - atd
 - atg
 - ati
 - atj
 - atq
 - att
 - auc
 - aui
 - auy
 - ava
 - avk
 - avn
 - avt
 - avu
 - awa
 - awb
 - awx
 - ayo
 - ayp
 - ayr
 - azb
 - azg
 - azj
 - azz
 - bak
 - bam
 - ban
 - bao
 - bar
 - bas
 - bav
 - bba
 - bbb
 - bbc
 - bbj
 - bbk
 - bbo
 - bbr
 - bch
 - bci
 - bcl
 - bco
 - bcw
 - bdd
 - bdh
 - bdq
 - bea
 - bef
 - bel
 - bem
 - ben
 - beq
 - bew
 - bex
 - bfd
 - bfo
 - bgr
 - bgs
 - bgt
 - bgz
 - bhg
 - bhl
 - bho
 - bhp
 - bhw
 - bhz
 - bib
 - big
 - bim
 - bin
 - bis
 - biu
 - biv
 - bjn
 - bjp
 - bjr
 - bjv
 - bkd
 - bkl
 - bkq
 - bku
 - bkv
 - bla
 - blh
 - blk
 - blw
 - blz
 - bmh
 - bmk
 - bmq
 - bmr
 - bmu
 - bmv
 - bno
 - bnp
 - boa
 - bod
 - boj
 - bom
 - bon
 - bos
 - bov
 - box
 - bpr
 - bps
 - bpy
 - bqc
 - bqj
 - bqp
 - bre
 - brh
 - bru
 - brx
 - bsc
 - bsn
 - bsp
 - bsq
 - bss
 - btd
 - bth
 - bts
 - btt
 - btx
 - bud
 - bug
 - buk
 - bul
 - bum
 - bus
 - bvc
 - bvd
 - bvr
 - bvz
 - bwd
 - bwi
 - bwq
 - bwu
 - bxh
 - bxr
 - byr
 - byv
 - byx
 - bzd
 - bzh
 - bzi
 - bzj
 - caa
 - cab
 - cac
 - caf
 - cag
 - cak
 - cao
 - cap
 - caq
 - car
 - cas
 - cat
 - cav
 - cax
 - cbc
 - cbi
 - cbk
 - cbr
 - cbs
 - cbt
 - cbu
 - cbv
 - cce
 - cco
 - ccp
 - ceb
 - ceg
 - cek
 - ces
 - cfm
 - cgc
 - cgg
 - cha
 - chd
 - che
 - chf
 - chj
 - chk
 - cho
 - chq
 - chr
 - chu
 - chv
 - chw
 - chz
 - cjk
 - cjo
 - cjp
 - cjs
 - cjv
 - ckb
 - cko
 - ckt
 - cle
 - clu
 - cly
 - cme
 - cmn
 - cmo
 - cmr
 - cnh
 - cni
 - cnk
 - cnl
 - cnt
 - cnw
 - coe
 - cof
 - cok
 - con
 - cop
 - cor
 - cos
 - cot
 - cou
 - cpa
 - cpb
 - cpc
 - cpu
 - cpy
 - crh
 - crj
 - crk
 - crl
 - crm
 - crn
 - crs
 - crt
 - crx
 - csb
 - csk
 - cso
 - csw
 - csy
 - cta
 - ctd
 - cto
 - ctp
 - ctu
 - cub
 - cuc
 - cui
 - cuk
 - cul
 - cut
 - cux
 - cwe
 - cwt
 - cya
 - cym
 - czt
 - daa
 - dad
 - daf
 - dag
 - dah
 - dak
 - dan
 - dar
 - ddg
 - ddn
 - ded
 - des
 - deu
 - dga
 - dgc
 - dgi
 - dgr
 - dgz
 - dhg
 - dhm
 - dhv
 - did
 - dig
 - dik
 - diq
 - dis
 - diu
 - div
 - dje
 - djk
 - djr
 - dks
 - dln
 - dng
 - dnj
 - dnw
 - dob
 - doi
 - dop
 - dos
 - dow
 - drg
 - dru
 - dsb
 - dtb
 - dtp
 - dts
 - dty
 - dua
 - due
 - dug
 - duo
 - dur
 - dwr
 - dww
 - dyi
 - dyo
 - dyu
 - dzo
 - ebk
 - efi
 - eka
 - ekk
 - eko
 - ell
 - emi
 - eml
 - emp
 - enb
 - enl
 - enm
 - enq
 - enx
 - epo
 - eri
 - ese
 - esi
 - esk
 - ess
 - esu
 - eto
 - etr
 - etu
 - eus
 - eve
 - ewe
 - ewo
 - ext
 - eza
 - faa
 - fad
 - fai
 - fal
 - fan
 - fao
 - far
 - fas
 - fat
 - ffm
 - fij
 - fil
 - fin
 - fit
 - fkv
 - fmu
 - fon
 - for
 - fra
 - frd
 - fro
 - frp
 - frr
 - fry
 - fub
 - fud
 - fue
 - fuf
 - fuh
 - fuq
 - fur
 - fuv
 - gaa
 - gag
 - gah
 - gai
 - gam
 - gaw
 - gaz
 - gbi
 - gbo
 - gbr
 - gcf
 - gcr
 - gde
 - gdg
 - gdn
 - gdr
 - geb
 - gej
 - gfk
 - ghs
 - gid
 - gil
 - giz
 - gjn
 - gkn
 - gla
 - gle
 - glg
 - glk
 - glv
 - gmh
 - gmv
 - gna
 - gnb
 - gnd
 - gng
 - gnn
 - gnw
 - goa
 - gof
 - gog
 - goh
 - gom
 - gor
 - gos
 - got
 - gqr
 - grc
 - grt
 - gso
 - gsw
 - gub
 - guc
 - gud
 - gug
 - guh
 - gui
 - guj
 - guk
 - gul
 - gum
 - gun
 - guo
 - guq
 - gur
 - guu
 - guw
 - gux
 - guz
 - gvc
 - gvf
 - gvl
 - gvn
 - gwi
 - gwr
 - gya
 - gym
 - gyr
 - hac
 - hae
 - hag
 - hak
 - hat
 - hav
 - haw
 - hay
 - hbo
 - hch
 - heb
 - heg
 - heh
 - her
 - hif
 - hig
 - hil
 - hin
 - hix
 - hla
 - hmo
 - hmr
 - hne
 - hnj
 - hnn
 - hns
 - hop
 - hot
 - hra
 - hrv
 - hrx
 - hsb
 - hto
 - hub
 - hui
 - hun
 - hus
 - huu
 - huv
 - hvn
 - hwc
 - hye
 - hyw
 - ian
 - iba
 - ibg
 - ibo
 - icr
 - ido
 - idu
 - ifa
 - ifb
 - ife
 - ifk
 - ifu
 - ify
 - ige
 - ign
 - ike
 - ikk
 - ikt
 - ikw
 - ilb
 - ile
 - ilo
 - imo
 - ina
 - inb
 - ind
 - inh
 - ino
 - iou
 - ipi
 - iqw
 - iri
 - irk
 - iry
 - isd
 - ish
 - isl
 - iso
 - ita
 - itv
 - ium
 - ivb
 - ivv
 - iws
 - ixl
 - izr
 - izz
 - jaa
 - jac
 - jae
 - jam
 - jav
 - jbo
 - jbu
 - jic
 - jiv
 - jmc
 - jpn
 - jra
 - jun
 - jvn
 - kaa
 - kab
 - kac
 - kak
 - kal
 - kam
 - kan
 - kao
 - kaq
 - kas
 - kat
 - kaz
 - kbc
 - kbd
 - kbh
 - kbm
 - kbo
 - kbp
 - kbq
 - kbr
 - kby
 - kca
 - kcg
 - kck
 - kdc
 - kde
 - kdh
 - kdi
 - kdj
 - kdl
 - kdr
 - kea
 - kei
 - kek
 - ken
 - keo
 - ker
 - kew
 - kez
 - kff
 - kgf
 - kgk
 - kgp
 - kgr
 - kha
 - khk
 - khm
 - khs
 - khz
 - kia
 - kij
 - kik
 - kin
 - kir
 - kiu
 - kix
 - kjb
 - kje
 - kjh
 - kjs
 - kkc
 - kki
 - kkj
 - kkl
 - kle
 - klt
 - klv
 - kmb
 - kmg
 - kmh
 - kmk
 - kmm
 - kmo
 - kmr
 - kms
 - kmu
 - kmy
 - knc
 - kne
 - knf
 - kng
 - knj
 - knk
 - kno
 - knv
 - knx
 - kny
 - kog
 - koi
 - koo
 - kor
 - kos
 - kpe
 - kpf
 - kpg
 - kpj
 - kpq
 - kpr
 - kpv
 - kpw
 - kpx
 - kpz
 - kqc
 - kqe
 - kqf
 - kql
 - kqn
 - kqo
 - kqp
 - kqs
 - kqw
 - kqy
 - krc
 - kri
 - krj
 - krl
 - kru
 - krx
 - ksb
 - ksc
 - ksd
 - ksf
 - ksh
 - ksj
 - ksp
 - ksr
 - kss
 - ksw
 - ktb
 - ktj
 - ktm
 - kto
 - ktu
 - ktz
 - kua
 - kub
 - kud
 - kue
 - kuj
 - kum
 - kup
 - kus
 - kvg
 - kvj
 - kvn
 - kwd
 - kwf
 - kwi
 - kwj
 - kwn
 - kwy
 - kxc
 - kxm
 - kxw
 - kyc
 - kyf
 - kyg
 - kyq
 - kyu
 - kyz
 - kze
 - kzf
 - kzj
 - lac
 - lad
 - lai
 - laj
 - lam
 - lao
 - lap
 - lat
 - lbb
 - lbe
 - lbj
 - lbk
 - lcm
 - lcp
 - ldi
 - ldn
 - lee
 - lef
 - leh
 - lem
 - leu
 - lew
 - lex
 - lez
 - lfn
 - lgg
 - lgl
 - lgm
 - lhi
 - lhu
 - lia
 - lid
 - lif
 - lij
 - lim
 - lin
 - lip
 - lis
 - lit
 - liv
 - ljp
 - lki
 - llb
 - lld
 - llg
 - lln
 - lmk
 - lmo
 - lmp
 - lnd
 - lob
 - loe
 - log
 - lok
 - lol
 - lom
 - loq
 - loz
 - lrc
 - lsi
 - lsm
 - ltg
 - ltz
 - lua
 - lub
 - luc
 - lud
 - lue
 - lug
 - lun
 - luo
 - lus
 - lvs
 - lwg
 - lwo
 - lww
 - lzh
 - maa
 - mad
 - maf
 - mag
 - mah
 - mai
 - maj
 - mak
 - mal
 - mam
 - maq
 - mar
 - mas
 - mau
 - mav
 - maw
 - maz
 - mbb
 - mbc
 - mbd
 - mbf
 - mbh
 - mbi
 - mbj
 - mbl
 - mbs
 - mbt
 - mca
 - mcb
 - mcd
 - mcf
 - mck
 - mcn
 - mco
 - mcp
 - mcq
 - mcu
 - mda
 - mdf
 - mdy
 - med
 - mee
 - mej
 - mek
 - men
 - meq
 - mer
 - met
 - meu
 - mev
 - mfe
 - mfg
 - mfh
 - mfi
 - mfk
 - mfq
 - mfy
 - mfz
 - mgc
 - mgh
 - mgo
 - mgr
 - mhi
 - mhl
 - mhr
 - mhw
 - mhx
 - mhy
 - mib
 - mic
 - mie
 - mif
 - mig
 - mih
 - mil
 - mim
 - min
 - mio
 - mip
 - miq
 - mir
 - mit
 - miy
 - miz
 - mjc
 - mjw
 - mkd
 - mkl
 - mkn
 - mks
 - mkz
 - mlh
 - mlp
 - mlt
 - mlu
 - mmn
 - mmo
 - mmx
 - mna
 - mnb
 - mnf
 - mni
 - mnk
 - mns
 - mnw
 - mnx
 - mny
 - moa
 - moc
 - mog
 - moh
 - mop
 - mor
 - mos
 - mox
 - mpg
 - mph
 - mpm
 - mpp
 - mps
 - mpt
 - mpx
 - mqb
 - mqj
 - mqy
 - mrg
 - mri
 - mrj
 - mrq
 - mrv
 - mrw
 - msb
 - msc
 - mse
 - msk
 - msy
 - mta
 - mtg
 - mti
 - mto
 - mtp
 - mua
 - mug
 - muh
 - mui
 - mup
 - mur
 - mus
 - mux
 - muy
 - mva
 - mvn
 - mvp
 - mwc
 - mwf
 - mwl
 - mwm
 - mwn
 - mwp
 - mwq
 - mwv
 - mww
 - mxb
 - mxp
 - mxq
 - mxt
 - mxv
 - mya
 - myb
 - myk
 - myu
 - myv
 - myw
 - myx
 - myy
 - mza
 - mzh
 - mzk
 - mzl
 - mzm
 - mzn
 - mzw
 - mzz
 - nab
 - naf
 - nah
 - nak
 - nap
 - naq
 - nas
 - nav
 - naw
 - nba
 - nbc
 - nbe
 - nbl
 - nbq
 - nbu
 - nca
 - nch
 - ncj
 - ncl
 - ncq
 - nct
 - ncu
 - ncx
 - ndc
 - nde
 - ndh
 - ndi
 - ndj
 - ndo
 - nds
 - ndz
 - neb
 - new
 - nfa
 - nfr
 - ngb
 - ngc
 - ngl
 - ngp
 - ngu
 - nhd
 - nhe
 - nhg
 - nhi
 - nhk
 - nho
 - nhr
 - nhu
 - nhw
 - nhx
 - nhy
 - nia
 - nif
 - nii
 - nij
 - nim
 - nin
 - nio
 - niu
 - niy
 - njb
 - njm
 - njn
 - njo
 - njz
 - nkf
 - nko
 - nld
 - nlg
 - nma
 - nmf
 - nmh
 - nmo
 - nmw
 - nmz
 - nnb
 - nng
 - nnh
 - nnl
 - nno
 - nnp
 - nnq
 - nnw
 - noa
 - nob
 - nod
 - nog
 - non
 - nop
 - not
 - nou
 - nov
 - nph
 - npi
 - npl
 - npo
 - npy
 - nqo
 - nre
 - nrf
 - nri
 - nrm
 - nsa
 - nse
 - nsm
 - nsn
 - nso
 - nss
 - nst
 - nsu
 - ntp
 - ntr
 - ntu
 - nuj
 - nus
 - nuy
 - nvm
 - nwb
 - nwi
 - nwx
 - nxd
 - nya
 - nyf
 - nyk
 - nyn
 - nyo
 - nyu
 - nyy
 - nza
 - nzi
 - nzm
 - obo
 - oci
 - ogo
 - ojb
 - oke
 - oku
 - okv
 - old
 - olo
 - omb
 - omw
 - ong
 - ons
 - ood
 - opm
 - orv
 - ory
 - oss
 - ota
 - otd
 - ote
 - otm
 - otn
 - oto
 - otq
 - ots
 - otw
 - oym
 - ozm
 - pab
 - pad
 - pag
 - pah
 - pam
 - pan
 - pao
 - pap
 - pau
 - pbb
 - pbc
 - pbi
 - pbt
 - pcd
 - pck
 - pcm
 - pdc
 - pdt
 - pem
 - pfe
 - pfl
 - phm
 - pib
 - pio
 - pir
 - pis
 - pjt
 - pkb
 - plg
 - pls
 - plt
 - plu
 - plw
 - pma
 - pmf
 - pmq
 - pms
 - pmx
 - pnb
 - pne
 - pnt
 - pny
 - poe
 - poh
 - poi
 - pol
 - pon
 - por
 - pos
 - pot
 - pov
 - poy
 - ppk
 - ppo
 - pps
 - prf
 - prg
 - pri
 - prq
 - pse
 - pss
 - ptp
 - ptu
 - pui
 - pwg
 - pwn
 - pww
 - pxm
 - qub
 - quc
 - quf
 - qug
 - quh
 - qul
 - qup
 - qus
 - quw
 - quy
 - quz
 - qva
 - qvc
 - qve
 - qvh
 - qvi
 - qvm
 - qvn
 - qvo
 - qvs
 - qvw
 - qvz
 - qwh
 - qxh
 - qxl
 - qxn
 - qxo
 - qxr
 - rad
 - rai
 - rap
 - rar
 - rav
 - raw
 - rcf
 - rej
 - rel
 - rgu
 - rhg
 - ria
 - rim
 - rjs
 - rkb
 - rmc
 - rme
 - rml
 - rmn
 - rmo
 - rmq
 - rmy
 - rnd
 - rng
 - rnl
 - roh
 - ron
 - roo
 - rop
 - row
 - rro
 - rtm
 - rub
 - rue
 - ruf
 - rug
 - run
 - rup
 - rus
 - rwo
 - sab
 - sag
 - sah
 - san
 - sas
 - sat
 - sba
 - sbd
 - sbe
 - sbl
 - sbs
 - sby
 - sck
 - scn
 - sco
 - sda
 - sdc
 - sdh
 - sdo
 - sdq
 - seh
 - ses
 - sey
 - sfw
 - sgb
 - sgc
 - sgh
 - sgs
 - sgw
 - sgz
 - shi
 - shk
 - shn
 - shp
 - shu
 - sid
 - sig
 - sil
 - sim
 - sin
 - sja
 - sjo
 - sju
 - skg
 - skr
 - sld
 - slk
 - sll
 - slv
 - sma
 - sme
 - smj
 - smk
 - sml
 - smn
 - smo
 - sms
 - smt
 - sna
 - snc
 - snd
 - snf
 - snn
 - snp
 - snw
 - sny
 - soe
 - som
 - sop
 - soq
 - sot
 - soy
 - spa
 - spl
 - spm
 - spp
 - sps
 - spy
 - srd
 - sri
 - srm
 - srn
 - srp
 - srq
 - srr
 - ssd
 - ssg
 - ssw
 - ssx
 - stn
 - stp
 - stq
 - sua
 - suc
 - sue
 - suk
 - sun
 - sur
 - sus
 - suz
 - swb
 - swc
 - swe
 - swg
 - swh
 - swk
 - swp
 - sxb
 - sxn
 - syb
 - syc
 - syl
 - szl
 - szy
 - tab
 - tac
 - tah
 - taj
 - tam
 - tap
 - taq
 - tar
 - tat
 - tav
 - taw
 - tay
 - tbc
 - tbg
 - tbk
 - tbl
 - tbo
 - tbw
 - tby
 - tbz
 - tca
 - tcc
 - tcf
 - tcs
 - tcy
 - tcz
 - ted
 - tee
 - tel
 - tem
 - teo
 - ter
 - tet
 - tew
 - tfr
 - tgk
 - tgo
 - tgp
 - tha
 - thk
 - thl
 - tif
 - tig
 - tih
 - tik
 - tim
 - tir
 - tiv
 - tiy
 - tke
 - tkl
 - tkr
 - tku
 - tlb
 - tlf
 - tlh
 - tlj
 - tll
 - tly
 - tmc
 - tmd
 - tna
 - tnc
 - tnk
 - tnn
 - tnp
 - tnr
 - tob
 - toc
 - tod
 - tog
 - toh
 - toi
 - toj
 - tok
 - ton
 - too
 - top
 - tos
 - tpa
 - tpi
 - tpm
 - tpp
 - tpt
 - tpw
 - tpz
 - tqo
 - trc
 - trn
 - tro
 - trp
 - trq
 - trs
 - trv
 - tsc
 - tsg
 - tsn
 - tso
 - tsw
 - tsz
 - ttc
 - tte
 - ttj
 - ttq
 - tuc
 - tue
 - tuf
 - tui
 - tuk
 - tul
 - tum
 - tuo
 - tur
 - tuv
 - tvk
 - tvl
 - twi
 - twu
 - twx
 - txq
 - txu
 - tyv
 - tzh
 - tzj
 - tzl
 - tzm
 - tzo
 - ubr
 - ubu
 - udm
 - udu
 - uig
 - ukr
 - umb
 - upv
 - ura
 - urb
 - urd
 - urh
 - uri
 - urk
 - urt
 - urw
 - ury
 - usa
 - usp
 - uth
 - uvh
 - uvl
 - uzn
 - uzs
 - vag
 - vap
 - var
 - vec
 - ven
 - vep
 - vid
 - vie
 - viv
 - vls
 - vmk
 - vmw
 - vmy
 - vol
 - vot
 - vro
 - vun
 - vut
 - waj
 - wal
 - wap
 - war
 - wat
 - way
 - wba
 - wbm
 - wbp
 - wed
 - wer
 - wes
 - wew
 - whg
 - whk
 - wib
 - wim
 - wiu
 - wln
 - wls
 - wlv
 - wlx
 - wmt
 - wmw
 - wnc
 - wnu
 - wob
 - wol
 - wos
 - wrk
 - wrs
 - wsg
 - wsk
 - wuu
 - wuv
 - wwa
 - xal
 - xav
 - xbi
 - xbr
 - xed
 - xho
 - xla
 - xmf
 - xmm
 - xmv
 - xnn
 - xog
 - xon
 - xrb
 - xsb
 - xsi
 - xsm
 - xsr
 - xsu
 - xtd
 - xtm
 - xtn
 - xuo
 - yaa
 - yad
 - yal
 - yam
 - yan
 - yao
 - yap
 - yaq
 - yat
 - yaz
 - ybb
 - yby
 - ycn
 - ydd
 - yim
 - yka
 - yle
 - yli
 - yml
 - yom
 - yon
 - yor
 - yrb
 - yre
 - yrk
 - yrl
 - yss
 - yua
 - yue
 - yuj
 - yup
 - yut
 - yuw
 - yuz
 - yva
 - zaa
 - zab
 - zac
 - zad
 - zae
 - zai
 - zam
 - zao
 - zar
 - zas
 - zat
 - zav
 - zaw
 - zca
 - zdj
 - zea
 - zgh
 - zia
 - ziw
 - zne
 - zom
 - zos
 - zpa
 - zpc
 - zpg
 - zpi
 - zpj
 - zpl
 - zpm
 - zpo
 - zpq
 - zpt
 - zpu
 - zpv
 - zpz
 - zsm
 - zsr
 - ztq
 - zty
 - zul
 - zyb
 - zyp
---

# mmBERT: A Modern Multilingual Encoder

[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
[![Paper](https://img.shields.io/badge/Paper-Arxiv-red)](https://arxiv.org/abs/2509.06888)
[![Model](https://img.shields.io/badge/🤗%20Hugging%20Face-Model-blue)](https://huggingface.co/jhu-clsp/mmBERT-base)
[![Collection](https://img.shields.io/badge/🤗%20Model%20Collection-blue)](https://huggingface.co/collections/jhu-clsp/mmbert-a-modern-multilingual-encoder-68b725831d7c6e3acc435ed4)
[![GitHub](https://img.shields.io/badge/GitHub-Code-black)](https://github.com/jhu-clsp/mmBERT)

> TL;DR: A state-of-the-art multilingual encoder trained on 3T+ tokens across 1800+ languages, introducing novel techniques for learning low-resource languages during the decay phase.

mmBERT is a modern multilingual encoder that significantly outperforms previous generation models like XLM-R on classification, embedding, and retrieval tasks. Built on the ModernBERT architecture with novel multilingual training innovations, mmBERT demonstrates that low-resource languages can be effectively learned during the decay phase of training. It is also significantly faster than any previous multilingual encoder.

## Table of Contents
- [Highlights](#highlights)
- [Quick Start](#quick-start)
- [Model Description](#model-description)
- [Novel Training Innovations](#novel-training-innovations)
- [Model Family](#model-family)
- [Training Data](#training-data)
- [Usage Examples](#usage-examples)
- [Fine-tuning Examples](#fine-tuning-examples)
- [Model Architecture](#model-architecture)
- [Citation](#citation)


## Quick Start

### Installation
```bash
pip install torch>=1.9.0
pip install transformers>=4.21.0
```

### Usage

```python
from transformers import AutoTokenizer, AutoModel

tokenizer = AutoTokenizer.from_pretrained("jhu-clsp/mmBERT-base")
model = AutoModel.from_pretrained("jhu-clsp/mmBERT-base")

inputs = tokenizer("Hello world", return_tensors="pt")
outputs = model(**inputs)
```

## Model Description

mmBERT represents the first significant advancement over XLM-R for massively multilingual encoder models. Key features include:

1. **Massive Language Coverage** - Trained on over 1800 languages with progressive inclusion strategy
2. **Modern Architecture** - Built on ModernBERT foundation with Flash Attention 2 and unpadding techniques
3. **Novel Training Recipe** - Introduces inverse mask scheduling and temperature sampling
4. **Open Training Data** - Complete 3T+ token dataset publicly available
5. **Decay Phase Innovation** - Demonstrates effective learning of low-resource languages in final training phase

The model uses bidirectional attention with masked language modeling objectives, optimized specifically for multilingual understanding and cross-lingual transfer.

## Novel Training Innovations

**Progressive Language Addition**: Start with 60 high-resource languages, expand to 110 mid-resource languages, then include all 1833 languages in decay phase.

**Inverse Mask Schedule**: Reduce mask ratio from 30% → 15% → 5% across training phases for progressively refined learning.

**Inverse Temperature Sampling**: Adjust multilingual sampling from high-resource bias (τ=0.7) to uniform sampling (τ=0.3).

**Model Merging**: Combine English-focused, high-resource, and all-language decay variants using TIES merging.

## Model Family

| Model | Total Params | Non-embed Params | Languages | Download |
|:------|:-------------|:------------------|:----------|:---------|
| [mmBERT-small](https://huggingface.co/jhu-clsp/mmBERT-small) | 140M | 42M | 1800+ | [![Download](https://img.shields.io/badge/🤗-Download-blue)](https://huggingface.co/jhu-clsp/mmBERT-small) |
| [mmBERT-base](https://huggingface.co/jhu-clsp/mmBERT-base) | 307M | 110M | 1800+ | [![Download](https://img.shields.io/badge/🤗-Download-blue)](https://huggingface.co/jhu-clsp/mmBERT-base) |

## Training Data

mmBERT training data is publicly available across different phases:

| Phase | Dataset | Tokens | Description |
|:------|:--------|:-------|:------------|
| Pre-training P1 | [mmbert-pretrain-p1](https://huggingface.co/datasets/jhu-clsp/mmbert-pretrain-p1-fineweb2-langs) | 2.3T | 60 languages, foundational training |
| Pre-training P2 | [mmbert-pretrain-p2](https://huggingface.co/datasets/jhu-clsp/mmBERT-pretrain-p2-fineweb2-remaining) | - | Extension data for pre-training phase |
| Pre-training P3 | [mmbert-pretrain-p3](https://huggingface.co/datasets/jhu-clsp/mmBERT-pretrain-p3-others) | - | Final pre-training data |
| Mid-training | [mmbert-midtraining](https://huggingface.co/datasets/jhu-clsp/mmbert-midtraining-data) | 600B | 110 languages, context extension to 8K |
| Decay Phase | [mmbert-decay](https://huggingface.co/datasets/jhu-clsp/mmbert-decay-data) | 100B | 1833 languages, premium quality |

**Data Sources**: Filtered DCLM (English), FineWeb2 (multilingual), FineWeb2-HQ (20 high-resource languages), Wikipedia (MegaWika), code repositories (StarCoder, ProLong), academic papers (ArXiv, PeS2o), and community discussions (StackExchange).

## Model Architecture

| Parameter | mmBERT-small | mmBERT-base |
|:----------|:-------------|:------------|
| Layers | 22 | 22 |
| Hidden Size | 384 | 768 |
| Intermediate Size | 1152 | 1152 |
| Attention Heads | 6 | 12 |
| Total Parameters | 140M | 307M |
| Non-embedding Parameters | 42M | 110M |
| Max Sequence Length | 8192 | 8192 |
| Vocabulary Size | 256,000 | 256,000 |
| Tokenizer | Gemma 2 | Gemma 2 |

## Usage Examples

### Masked Language Modeling

```python
from transformers import AutoTokenizer, AutoModelForMaskedLM
import torch

tokenizer = AutoTokenizer.from_pretrained("jhu-clsp/mmBERT-base")
model = AutoModelForMaskedLM.from_pretrained("jhu-clsp/mmBERT-base")

def predict_masked_token(text):
    inputs = tokenizer(text, return_tensors="pt")
    with torch.no_grad():
        outputs = model(**inputs)
    
    mask_indices = torch.where(inputs["input_ids"] == tokenizer.mask_token_id)
    predictions = outputs.logits[mask_indices]
    top_tokens = torch.topk(predictions, 5, dim=-1)
    
    return [tokenizer.decode(token) for token in top_tokens.indices[0]]

# Works across languages
texts = [
    "The capital of France is <mask>.",
    "La capital de España es <mask>.",
    "Die Hauptstadt von Deutschland ist <mask>."
]

for text in texts:
    predictions = predict_masked_token(text)
    print(f"Text: {text}")
    print(f"Predictions: {predictions}")
```

### Cross-lingual Embeddings

```python
from transformers import AutoTokenizer, AutoModel
import torch
from sklearn.metrics.pairwise import cosine_similarity

tokenizer = AutoTokenizer.from_pretrained("jhu-clsp/mmBERT-base")
model = AutoModel.from_pretrained("jhu-clsp/mmBERT-base")

def get_embeddings(texts):
    inputs = tokenizer(texts, padding=True, truncation=True, return_tensors="pt")
    
    with torch.no_grad():
        outputs = model(**inputs)
        embeddings = outputs.last_hidden_state.mean(dim=1)
    
    return embeddings.numpy()

multilingual_texts = [
    "Artificial intelligence is transforming technology",
    "La inteligencia artificial está transformando la tecnología",
    "L'intelligence artificielle transforme la technologie", 
    "人工智能正在改变技术"
]

embeddings = get_embeddings(multilingual_texts)
similarities = cosine_similarity(embeddings)
print("Cross-lingual similarity matrix:")
print(similarities)
```

## Fine-tuning Examples

### Dense Retrieval with Sentence Transformers

<details>
<summary>Click to expand dense retrieval fine-tuning example</summary>

```python
import argparse
from datasets import load_dataset
from sentence_transformers import (
    SentenceTransformer,
    SentenceTransformerTrainer,
    SentenceTransformerTrainingArguments,
)
from sentence_transformers.evaluation import TripletEvaluator
from sentence_transformers.losses import CachedMultipleNegativesRankingLoss
from sentence_transformers.training_args import BatchSamplers

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--lr", type=float, default=8e-5)
    parser.add_argument("--model_name", type=str, default="jhu-clsp/mmBERT-base")
    args = parser.parse_args()
    
    lr = args.lr
    model_name = args.model_name
    model_shortname = model_name.split("/")[-1]

    model = SentenceTransformer(model_name)

    dataset = load_dataset(
        "sentence-transformers/msmarco-co-condenser-margin-mse-sym-mnrl-mean-v1",
        "triplet-hard",
        split="train",
    )
    dataset_dict = dataset.train_test_split(test_size=1_000, seed=12)
    train_dataset = dataset_dict["train"].select(range(1_250_000))
    eval_dataset = dataset_dict["test"]

    loss = CachedMultipleNegativesRankingLoss(model, mini_batch_size=16)
    run_name = f"{model_shortname}-DPR-{lr}"
    
    training_args = SentenceTransformerTrainingArguments(
        output_dir=f"output/{model_shortname}/{run_name}",
        num_train_epochs=1,
        per_device_train_batch_size=512,
        per_device_eval_batch_size=512,
        warmup_ratio=0.05,
        fp16=False,
        bf16=True,
        batch_sampler=BatchSamplers.NO_DUPLICATES,
        learning_rate=lr,
        save_strategy="steps",
        save_steps=500,
        save_total_limit=2,
        logging_steps=500,
        run_name=run_name,
    )

    dev_evaluator = TripletEvaluator(
        anchors=eval_dataset["query"],
        positives=eval_dataset["positive"],
        negatives=eval_dataset["negative"],
        name="msmarco-co-condenser-dev",
    )
    dev_evaluator(model)

    trainer = SentenceTransformerTrainer(
        model=model,
        args=training_args,
        train_dataset=train_dataset,
        eval_dataset=eval_dataset,
        loss=loss,
        evaluator=dev_evaluator,
    )
    trainer.train()

    model.save_pretrained(f"output/{model_shortname}/{run_name}/final")
    model.push_to_hub(run_name, private=False)

if __name__ == "__main__":
    main()
```

</details>

### Cross-lingual Classification

<details>
<summary>Click to expand multilingual classification fine-tuning example</summary>

```python
from transformers import (
    AutoTokenizer,
    AutoModelForSequenceClassification, 
    TrainingArguments,
    Trainer
)
from datasets import load_dataset
import numpy as np
from sklearn.metrics import accuracy_score, f1_score

def compute_metrics(eval_pred):
    predictions, labels = eval_pred
    predictions = np.argmax(predictions, axis=1)
    return {
        'accuracy': accuracy_score(labels, predictions),
        'f1': f1_score(labels, predictions, average='weighted')
    }

def main():
    model_name = "jhu-clsp/mmBERT-base"
    
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    model = AutoModelForSequenceClassification.from_pretrained(
        model_name,
        num_labels=3
    )
    
    dataset = load_dataset("xnli", "all_languages")
    
    def tokenize_function(examples):
        texts = [f"{p} {tokenizer.sep_token} {h}" 
                for p, h in zip(examples["premise"], examples["hypothesis"])]
        
        return tokenizer(
            texts,
            truncation=True,
            padding=True,
            max_length=512
        )
    
    train_dataset = dataset["train"].map(tokenize_function, batched=True)
    eval_dataset = dataset["validation"].map(tokenize_function, batched=True)
    
    training_args = TrainingArguments(
        output_dir="./mmbert-xnli",
        learning_rate=3e-5,
        per_device_train_batch_size=32,
        per_device_eval_batch_size=32,
        num_train_epochs=3,
        weight_decay=0.01,
        evaluation_strategy="epoch",
        save_strategy="epoch",
        load_best_model_at_end=True,
        metric_for_best_model="f1",
        greater_is_better=True,
    )
    
    trainer = Trainer(
        model=model,
        args=training_args,
        train_dataset=train_dataset,
        eval_dataset=eval_dataset,
        compute_metrics=compute_metrics,
    )
    
    trainer.train()

if __name__ == "__main__":
    main()
```

</details>

### Multilingual Reranking

<details>
<summary>Click to expand multilingual reranking fine-tuning example</summary>

```python
import logging
from datasets import load_dataset
from sentence_transformers.cross_encoder import (
    CrossEncoder,
    CrossEncoderModelCardData,
    CrossEncoderTrainer,
    CrossEncoderTrainingArguments,
)
from sentence_transformers.cross_encoder.evaluation import CrossEncoderNanoBEIREvaluator
from sentence_transformers.cross_encoder.losses import BinaryCrossEntropyLoss
from sentence_transformers.util import mine_hard_negatives
from sentence_transformers import SentenceTransformer
import torch

def main():
    model_name = "jhu-clsp/mmBERT-base"
    train_batch_size = 32
    num_epochs = 2
    num_hard_negatives = 7

    model = CrossEncoder(
        model_name,
        model_card_data=CrossEncoderModelCardData(
            language="multilingual",
            license="mit",
        ),
    )
    
    full_dataset = load_dataset("sentence-transformers/gooaq", split="train").select(range(50_000))
    dataset_dict = full_dataset.train_test_split(test_size=1_000, seed=42)
    train_dataset = dataset_dict["train"]
    eval_dataset = dataset_dict["test"]

    embedding_model = SentenceTransformer("sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", device="cpu")
    hard_train_dataset = mine_hard_negatives(
        train_dataset,
        embedding_model,
        num_negatives=num_hard_negatives,
        margin=0,
        range_min=0,
        range_max=100,
        sampling_strategy="top",
        batch_size=2048,
        output_format="labeled-pair",
        use_faiss=True,
    )

    loss = BinaryCrossEntropyLoss(model=model, pos_weight=torch.tensor(num_hard_negatives))

    nano_beir_evaluator = CrossEncoderNanoBEIREvaluator(
        dataset_names=["msmarco", "nfcorpus", "nq"],
        batch_size=train_batch_size,
    )

    args = CrossEncoderTrainingArguments(
        output_dir="./mmbert-reranker",
        num_train_epochs=num_epochs,
        per_device_train_batch_size=train_batch_size,
        per_device_eval_batch_size=train_batch_size,
        learning_rate=2e-5,
        warmup_ratio=0.1,
        fp16=False,
        bf16=True,
        dataloader_num_workers=4,
        load_best_model_at_end=True,
        metric_for_best_model="eval_msmarco_ndcg@10",
        eval_strategy="steps",
        eval_steps=1000,
        save_strategy="steps",
        save_steps=1000,
        save_total_limit=2,
        logging_steps=200,
        seed=42,
    )

    trainer = CrossEncoderTrainer(
        model=model,
        args=args,
        train_dataset=hard_train_dataset,
        loss=loss,
        evaluator=nano_beir_evaluator,
    )
    trainer.train()

    model.save_pretrained("./mmbert-reranker/final")

if __name__ == "__main__":
    main()
```

</details>

## Training Data

mmBERT was trained on a carefully curated 3T+ token multilingual dataset:

| Phase | Dataset | Description |
|:------|:--------|:------------|
| [Pre-training P1](https://huggingface.co/datasets/jhu-clsp/mmbert-pretrain-p1-fineweb2-langs) | 2.3T tokens | 60 languages, diverse data mixture |
| [Pre-training P2](https://huggingface.co/datasets/jhu-clsp/mmbert-pretrain-p2-fineweb2-langs) | - | Extension data for pre-training |
| [Pre-training P3](https://huggingface.co/datasets/jhu-clsp/mmbert-pretrain-p3-fineweb2-langs) | - | Final pre-training data |
| [Mid-training](https://huggingface.co/datasets/jhu-clsp/mmbert-midtraining-data) | 600B tokens | 110 languages, context extension |
| [Decay Phase](https://huggingface.co/datasets/jhu-clsp/mmbert-decay-data) | 100B tokens | 1833 languages, premium quality |

**Primary Sources:**
- **Filtered DCLM**: High-quality English content
- **FineWeb2**: Broad multilingual web coverage (1800+ languages)
- **FineWeb2-HQ**: Filtered subset of 20 high-resource languages
- **Code**: StarCoder and ProLong repositories
- **Academic**: ArXiv papers and PeS2o scientific content
- **Reference**: Wikipedia (MegaWika) and textbooks
- **Community**: StackExchange discussions


## Citation

If you use mmBERT in your research, please cite our work:

```bibtex
@misc{marone2025mmbertmodernmultilingualencoder,
      title={mmBERT: A Modern Multilingual Encoder with Annealed Language Learning}, 
      author={Marc Marone and Orion Weller and William Fleshman and Eugene Yang and Dawn Lawrie and Benjamin Van Durme},
      year={2025},
      eprint={2509.06888},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2509.06888}, 
}
```
"""