Upload folder using huggingface_hub

.ipynb_checkpoints/README-checkpoint.md

@@ -23,13 +23,13 @@ model-index:
     metrics:
     - name: BLEU
       type: bleu
-      value: 29.9
+      value: 30.0
     - name: CHRF
       type: chrf
       value: 58.42
     - name: COMET
       type: comet
-      value: 86.59
+      value: 86.72
 ---
 
 
@@ -56,7 +56,7 @@ Give it a try before downloading here: https://huggingface.co/spaces/quickmt/Qui
 ## Model Information
 
 * Trained using [`eole`](https://github.com/eole-nlp/eole) 
-* 200M parameter transformer 'big' with 8 encoder layers and 2 decoder layers
+* 200M parameter seq2seq transformer
 * 32k separate Sentencepiece vocabs
 * Exported for fast inference to [CTranslate2](https://github.com/OpenNMT/CTranslate2) format
 * The pytorch model (for use with [`eole`](https://github.com/eole-nlp/eole)) is available in this repository in the `eole-model` folder
@@ -111,7 +111,7 @@ The model is in `ctranslate2` format, and the tokenizers are `sentencepiece`, so
 
 |                                  |   bleu |   chrf2 |   comet22 |   Time (s) |
 |:---------------------------------|-------:|--------:|----------:|-----------:|
-| quickmt/quickmt-zh-en            |  29.9  |   58.42 |     86.59 |       1.22 |
+| quickmt/quickmt-zh-en            |  30.0  |   58.42 |     86.72 |       1.10 |
 | Helsinki-NLP/opus-mt-zh-en       |  22.99 |   53.98 |     84.6  |       3.73 |
 | facebook/nllb-200-distilled-600M |  26.02 |   55.27 |     85.1  |      21.69 |
 | facebook/nllb-200-distilled-1.3B |  28.61 |   57.43 |     86.22 |      37.55 |

.ipynb_checkpoints/eole-config-checkpoint.yaml

@@ -5,7 +5,7 @@ seed: 1234
 report_every: 100
 valid_metrics: ["BLEU"]
 tensorboard: true
-tensorboard_log_dir: tensorboard
+tensorboard_log_dir: tensorboard_small
 
 ### Vocab
 src_vocab: zh.eole.vocab
@@ -18,9 +18,9 @@ n_sample: 0
 
 data:
     corpus_1:
-        path_src: hf://quickmt/quickmt-train.is-en/zh
-        path_tgt: hf://quickmt/quickmt-train.is-en/en
-        path_sco: hf://quickmt/quickmt-train.is-en/sco
+        path_src: hf://quickmt/quickmt-train.zh-en/zh
+        path_tgt: hf://quickmt/quickmt-train.zh-en/en
+        path_sco: hf://quickmt/quickmt-train.zh-en/sco
         weight: 2
     corpus_2:
         path_src: hf://quickmt/newscrawl2024-en-backtranslated-zh/zh
@@ -57,7 +57,7 @@ training:
     world_size: 1
     gpu_ranks: [0]
 
-    # Batching
+    # Batching 10240
     batch_type: "tokens"
     batch_size: 6000
     valid_batch_size: 2048
@@ -75,9 +75,9 @@ training:
     adam_beta2: 0.998
 
     # Data loading
-    bucket_size: 128000
+    bucket_size: 256000
     num_workers: 4
-    prefetch_factor: 32
+    prefetch_factor: 64
     
     # Hyperparams
     dropout_steps: [0]
@@ -92,15 +92,20 @@ training:
 model:
     architecture: "transformer"
     share_embeddings: false
-    share_decoder_embeddings: true
-    hidden_size: 1024
+    share_decoder_embeddings: false
+    add_estimator: false
+    add_ffnbias: true
+    add_qkvbias: false
+    layer_norm: standard
+    mlp_activation_fn: gelu
+    hidden_size: 768
     encoder:
-        layers: 8
+        layers: 12
     decoder:
         layers: 2
     heads: 8
     transformer_ff: 4096
     embeddings:
-        word_vec_size: 1024
+        word_vec_size: 768
         position_encoding_type: "SinusoidalInterleaved"
 

README.md

@@ -23,13 +23,13 @@ model-index:
     metrics:
     - name: BLEU
       type: bleu
-      value: 29.9
+      value: 30.0
     - name: CHRF
       type: chrf
       value: 58.42
     - name: COMET
       type: comet
-      value: 86.59
+      value: 86.72
 ---
 
 
@@ -56,7 +56,7 @@ Give it a try before downloading here: https://huggingface.co/spaces/quickmt/Qui
 ## Model Information
 
 * Trained using [`eole`](https://github.com/eole-nlp/eole) 
-* 200M parameter transformer 'big' with 8 encoder layers and 2 decoder layers
+* 200M parameter seq2seq transformer
 * 32k separate Sentencepiece vocabs
 * Exported for fast inference to [CTranslate2](https://github.com/OpenNMT/CTranslate2) format
 * The pytorch model (for use with [`eole`](https://github.com/eole-nlp/eole)) is available in this repository in the `eole-model` folder
@@ -111,7 +111,7 @@ The model is in `ctranslate2` format, and the tokenizers are `sentencepiece`, so
 
 |                                  |   bleu |   chrf2 |   comet22 |   Time (s) |
 |:---------------------------------|-------:|--------:|----------:|-----------:|
-| quickmt/quickmt-zh-en            |  29.9  |   58.42 |     86.59 |       1.22 |
+| quickmt/quickmt-zh-en            |  30.0  |   58.42 |     86.72 |       1.10 |
 | Helsinki-NLP/opus-mt-zh-en       |  22.99 |   53.98 |     84.6  |       3.73 |
 | facebook/nllb-200-distilled-600M |  26.02 |   55.27 |     85.1  |      21.69 |
 | facebook/nllb-200-distilled-1.3B |  28.61 |   57.43 |     86.22 |      37.55 |

eole-config.yaml

@@ -5,7 +5,7 @@ seed: 1234
 report_every: 100
 valid_metrics: ["BLEU"]
 tensorboard: true
-tensorboard_log_dir: tensorboard
+tensorboard_log_dir: tensorboard_small
 
 ### Vocab
 src_vocab: zh.eole.vocab
@@ -18,9 +18,9 @@ n_sample: 0
 
 data:
     corpus_1:
-        path_src: hf://quickmt/quickmt-train.is-en/zh
-        path_tgt: hf://quickmt/quickmt-train.is-en/en
-        path_sco: hf://quickmt/quickmt-train.is-en/sco
+        path_src: hf://quickmt/quickmt-train.zh-en/zh
+        path_tgt: hf://quickmt/quickmt-train.zh-en/en
+        path_sco: hf://quickmt/quickmt-train.zh-en/sco
         weight: 2
     corpus_2:
         path_src: hf://quickmt/newscrawl2024-en-backtranslated-zh/zh
@@ -57,7 +57,7 @@ training:
     world_size: 1
     gpu_ranks: [0]
 
-    # Batching
+    # Batching 10240
     batch_type: "tokens"
     batch_size: 6000
     valid_batch_size: 2048
@@ -75,9 +75,9 @@ training:
     adam_beta2: 0.998
 
     # Data loading
-    bucket_size: 128000
+    bucket_size: 256000
     num_workers: 4
-    prefetch_factor: 32
+    prefetch_factor: 64
     
     # Hyperparams
     dropout_steps: [0]
@@ -92,15 +92,20 @@ training:
 model:
     architecture: "transformer"
     share_embeddings: false
-    share_decoder_embeddings: true
-    hidden_size: 1024
+    share_decoder_embeddings: false
+    add_estimator: false
+    add_ffnbias: true
+    add_qkvbias: false
+    layer_norm: standard
+    mlp_activation_fn: gelu
+    hidden_size: 768
     encoder:
-        layers: 8
+        layers: 12
     decoder:
         layers: 2
     heads: 8
     transformer_ff: 4096
     embeddings:
-        word_vec_size: 1024
+        word_vec_size: 768
         position_encoding_type: "SinusoidalInterleaved"
 

eole-model/config.json

@@ -1,109 +1,147 @@
 {
-  "n_sample": 0,
-  "share_vocab": false,
-  "report_every": 100,
-  "tgt_vocab_size": 32000,
-  "tensorboard_log_dir": "tensorboard",
-  "tensorboard_log_dir_dated": "tensorboard/Nov-28_15-33-54",
   "valid_metrics": [
     "BLEU"
   ],
-  "src_vocab": "zh.eole.vocab",
-  "tensorboard": true,
-  "seed": 1234,
-  "tgt_vocab": "en.eole.vocab",
-  "vocab_size_multiple": 8,
   "transforms": [
     "sentencepiece",
     "filtertoolong"
   ],
   "src_vocab_size": 32000,
+  "tensorboard": true,
+  "n_sample": 0,
   "overwrite": true,
+  "vocab_size_multiple": 8,
+  "share_vocab": false,
   "save_data": "data",
+  "seed": 1234,
+  "tensorboard_log_dir_dated": "tensorboard/Dec-08_21-16-22",
+  "tgt_vocab": "en.eole.vocab",
+  "src_vocab": "zh.eole.vocab",
+  "tgt_vocab_size": 32000,
+  "report_every": 100,
+  "tensorboard_log_dir": "tensorboard",
   "training": {
-    "num_workers": 0,
-    "label_smoothing": 0.1,
-    "accum_count": [
-      20
-    ],
-    "valid_steps": 5000,
     "gpu_ranks": [
       0
     ],
-    "accum_steps": [
-      0
-    ],
-    "warmup_steps": 5000,
-    "world_size": 1,
-    "batch_size_multiple": 8,
-    "optim": "adamw",
+    "keep_checkpoint": 4,
+    "decay_method": "noam",
+    "valid_steps": 5000,
+    "save_checkpoint_steps": 5000,
+    "model_path": "quickmt-zh-en-tiny-eole-model",
+    "adam_beta2": 0.998,
+    "num_workers": 0,
     "normalization": "tokens",
-    "max_grad_norm": 0.0,
-    "bucket_size": 128000,
+    "learning_rate": 3.0,
+    "batch_size": 6000,
+    "compute_dtype": "torch.float16",
+    "warmup_steps": 5000,
     "dropout": [
       0.1
     ],
-    "adam_beta2": 0.998,
-    "model_path": "quickmt-zh-en-eole-model",
-    "batch_size": 6000,
-    "batch_type": "tokens",
-    "compute_dtype": "torch.float16",
-    "save_checkpoint_steps": 5000,
-    "keep_checkpoint": 4,
-    "learning_rate": 3.0,
-    "prefetch_factor": 32,
-    "dropout_steps": [
+    "attention_dropout": [
+      0.1
+    ],
+    "world_size": 1,
+    "accum_steps": [
       0
     ],
-    "train_steps": 200000,
-    "decay_method": "noam",
-    "average_decay": 0.0001,
+    "accum_count": [
+      20
+    ],
+    "prefetch_factor": 64,
     "valid_batch_size": 2048,
+    "average_decay": 0.0001,
+    "dropout_steps": [
+      0
+    ],
+    "max_grad_norm": 0.0,
+    "batch_type": "tokens",
+    "bucket_size": 256000,
+    "label_smoothing": 0.1,
     "param_init_method": "xavier_uniform",
-    "attention_dropout": [
-      0.1
-    ]
+    "batch_size_multiple": 8,
+    "optim": "adamw",
+    "train_steps": 200000
   },
-  "transforms_configs": {
-    "sentencepiece": {
-      "src_subword_model": "${MODEL_PATH}/zh.spm.model",
-      "tgt_subword_model": "${MODEL_PATH}/en.spm.model"
+  "model": {
+    "mlp_activation_fn": "gelu",
+    "layer_norm": "standard",
+    "hidden_size": 768,
+    "add_qkvbias": false,
+    "transformer_ff": 4096,
+    "add_estimator": false,
+    "share_decoder_embeddings": false,
+    "add_ffnbias": true,
+    "architecture": "transformer",
+    "heads": 8,
+    "position_encoding_type": "SinusoidalInterleaved",
+    "share_embeddings": false,
+    "encoder": {
+      "mlp_activation_fn": "gelu",
+      "encoder_type": "transformer",
+      "n_positions": null,
+      "layer_norm": "standard",
+      "hidden_size": 768,
+      "add_qkvbias": false,
+      "transformer_ff": 4096,
+      "layers": 12,
+      "src_word_vec_size": 768,
+      "add_ffnbias": true,
+      "heads": 8,
+      "position_encoding_type": "SinusoidalInterleaved"
     },
-    "filtertoolong": {
-      "src_seq_length": 256,
-      "tgt_seq_length": 256
+    "embeddings": {
+      "position_encoding_type": "SinusoidalInterleaved",
+      "word_vec_size": 768,
+      "tgt_word_vec_size": 768,
+      "src_word_vec_size": 768
+    },
+    "decoder": {
+      "mlp_activation_fn": "gelu",
+      "layer_norm": "standard",
+      "n_positions": null,
+      "hidden_size": 768,
+      "decoder_type": "transformer",
+      "add_qkvbias": false,
+      "layers": 2,
+      "transformer_ff": 4096,
+      "tgt_word_vec_size": 768,
+      "add_ffnbias": true,
+      "heads": 8,
+      "position_encoding_type": "SinusoidalInterleaved"
     }
   },
   "data": {
     "corpus_1": {
+      "path_src": "train.zh",
+      "path_tgt": "train.en",
       "weight": 2,
       "transforms": [
         "sentencepiece",
         "filtertoolong"
       ],
-      "path_align": null,
-      "path_src": "train.zh",
-      "path_tgt": "train.en"
+      "path_align": null
     },
     "corpus_2": {
+      "path_src": "/home/mark/mt/data/newscrawl.backtrans.zh",
+      "path_tgt": "/home/mark/mt/data/newscrawl.2024.en",
       "weight": 1,
       "transforms": [
         "sentencepiece",
         "filtertoolong"
       ],
-      "path_align": null,
-      "path_src": "/home/mark/mt/data/newscrawl.backtrans.zh",
-      "path_tgt": "/home/mark/mt/data/newscrawl.2024.en"
+      "path_align": null
     },
     "corpus_3": {
+      "path_src": "/home/mark/mt/data/madlad.backtrans.zh",
+      "path_tgt": "/home/mark/mt/data/madlad.en",
       "weight": 2,
       "transforms": [
         "sentencepiece",
         "filtertoolong"
       ],
-      "path_align": null,
-      "path_src": "/home/mark/mt/data/madlad.backtrans.zh",
-      "path_tgt": "/home/mark/mt/data/madlad.en"
+      "path_align": null
     },
     "valid": {
       "path_src": "valid.zh",
@@ -111,43 +149,18 @@
         "sentencepiece",
         "filtertoolong"
       ],
-      "path_tgt": "valid.en",
-      "path_align": null
+      "path_align": null,
+      "path_tgt": "valid.en"
     }
   },
-  "model": {
-    "hidden_size": 1024,
-    "position_encoding_type": "SinusoidalInterleaved",
-    "share_embeddings": false,
-    "architecture": "transformer",
-    "heads": 8,
-    "share_decoder_embeddings": true,
-    "transformer_ff": 4096,
-    "decoder": {
-      "hidden_size": 1024,
-      "layers": 2,
-      "position_encoding_type": "SinusoidalInterleaved",
-      "tgt_word_vec_size": 1024,
-      "n_positions": null,
-      "heads": 8,
-      "decoder_type": "transformer",
-      "transformer_ff": 4096
-    },
-    "embeddings": {
-      "src_word_vec_size": 1024,
-      "word_vec_size": 1024,
-      "position_encoding_type": "SinusoidalInterleaved",
-      "tgt_word_vec_size": 1024
+  "transforms_configs": {
+    "sentencepiece": {
+      "src_subword_model": "${MODEL_PATH}/zh.spm.model",
+      "tgt_subword_model": "${MODEL_PATH}/en.spm.model"
     },
-    "encoder": {
-      "hidden_size": 1024,
-      "encoder_type": "transformer",
-      "src_word_vec_size": 1024,
-      "layers": 8,
-      "position_encoding_type": "SinusoidalInterleaved",
-      "n_positions": null,
-      "heads": 8,
-      "transformer_ff": 4096
+    "filtertoolong": {
+      "src_seq_length": 256,
+      "tgt_seq_length": 256
     }
   }
 }

eole-model/model.00.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ce0b3dfe2ef4c9f6b93f969f27f5d5cf38432e4a7bcd144577c8583209bb701a
-size 840314816
+oid sha256:708eecb88e180b1e8d1bb240bf51d27c324d67d00dde6e3fc3551ef6b6299c4c
+size 829569112

model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:88ef37879afce2d5f0bdf4c53073aab30967f178f0a0fa2eed7c98160270b06a
-size 409915789
+oid sha256:e7cac9c2bc585e476d8fe47af11ed76723d4b5cf69ccef8f5ff821fb3ec05000
+size 407101843