Sherpa-CTC (en)

.gitattributes

@@ -39,3 +39,5 @@ en/sherpa-onnx-nemo-ctc-en-conformer-small/test_wavs/0.wav filter=lfs diff=lfs m
 en/sherpa-onnx-nemo-ctc-en-conformer-small/test_wavs/1.wav filter=lfs diff=lfs merge=lfs -text
 en/sherpa-onnx-nemo-ctc-en-conformer-medium/test_wavs/0.wav filter=lfs diff=lfs merge=lfs -text
 en/sherpa-onnx-nemo-ctc-en-conformer-medium/test_wavs/1.wav filter=lfs diff=lfs merge=lfs -text
+en/sherpa-onnx-nemo-ctc-en-conformer-large/test_wavs/0.wav filter=lfs diff=lfs merge=lfs -text
+en/sherpa-onnx-nemo-ctc-en-conformer-large/test_wavs/1.wav filter=lfs diff=lfs merge=lfs -text

en/sherpa-onnx-nemo-ctc-en-conformer-large/.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

en/sherpa-onnx-nemo-ctc-en-conformer-large/README.md

@@ -0,0 +1,21 @@
+---
+license: apache-2.0
+---
+
+# Introduction
+
+This repo contains torchscript model of `stt_en_conformer_ctc_large` from NeMo.
+
+See https://registry.ngc.nvidia.com/orgs/nvidia/teams/nemo/models/stt_en_conformer_ctc_large
+
+The following code is used to obtain `model.onnx` and `tokens.txt`:
+
+```python3
+m = nemo_asr.models.EncDecCTCModelBPE.from_pretrained('stt_en_conformer_ctc_large')
+m.export('model.onnx')
+
+with open('tokens.txt', 'w') as f:
+  for i, s in enumerate(m.decoder.vocabulary):
+    f.write(f"{s} {i}\n")
+  f.write(f"<blk> {i+1}\n")
+`

en/sherpa-onnx-nemo-ctc-en-conformer-large/add-model-metadata.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+
+# Copyright (c)  2023  Xiaomi Corporation
+# Author: Fangjun Kuang
+
+from typing import Dict
+
+import numpy as np
+import onnx
+
+
+def get_vocab_size():
+    with open("tokens.txt") as f:
+        return len(f.readlines())
+
+
+def add_meta_data(filename: str, meta_data: Dict[str, str]):
+    """Add meta data to an ONNX model. It is changed in-place.
+
+    Args:
+      filename:
+        Filename of the ONNX model to be changed.
+      meta_data:
+        Key-value pairs.
+    """
+    model = onnx.load(filename)
+    for key, value in meta_data.items():
+        meta = model.metadata_props.add()
+        meta.key = key
+        meta.value = value
+
+    onnx.save(model, filename)
+    print(f"Updated {filename}")
+
+
+def main():
+    vocab_size = get_vocab_size()
+    # 8 for citrinet
+    # 4 for conformer ctc
+    subsampling_factor = 4
+
+    meta_data = {
+        "vocab_size": str(vocab_size),
+        "normalize_type": "per_feature",
+        "subsampling_factor": str(subsampling_factor),
+        "model_type": "EncDecCTCModelBPE",
+        "version": "1",
+        "model_author": "nemo",
+        "comment": "https://registry.ngc.nvidia.com/orgs/nvidia/teams/nemo/models/stt_en_conformer_ctc_small",
+    }
+    add_meta_data("model.onnx", meta_data)
+
+
+if __name__ == "__main__":
+    main()

en/sherpa-onnx-nemo-ctc-en-conformer-large/model.int8.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7055703cc9c5dc787706d10d4c0260a903695d4958933a97f64352902320c5c5
+size 169392184

en/sherpa-onnx-nemo-ctc-en-conformer-large/model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44e0d51556e5d99fdfff481660bef8d5a1dcd3cd1b2f34ec55c2f421a296a66a
+size 532287873

en/sherpa-onnx-nemo-ctc-en-conformer-large/quantize-model.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+
+import onnx
+from onnxruntime.quantization import QuantType, quantize_dynamic
+
+
+def main():
+    onnx_model = onnx.load("model.onnx")
+    quantize_dynamic(
+        model_input="model.onnx",
+        model_output="model.int8.onnx",
+        per_channel=True,
+        weight_type=QuantType.QUInt8,
+    )
+
+
+if __name__ == "__main__":
+    main()

en/sherpa-onnx-nemo-ctc-en-conformer-large/source.txt

@@ -0,0 +1 @@
+https://huggingface.co/csukuangfj/sherpa-onnx-nemo-ctc-en-conformer-large

en/sherpa-onnx-nemo-ctc-en-conformer-large/test.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+
+# Copyright (c)  2023  Xiaomi Corporation
+# Author: Fangjun Kuang
+
+import kaldi_native_fbank as knf
+import itertools
+import librosa
+import numpy as np
+import onnxruntime as ort
+
+
+def compute_feat(filename):
+    sample_rate = 16000
+    samples, _ = librosa.load(filename, sr=sample_rate)
+    opts = knf.FbankOptions()
+    opts.frame_opts.dither = 0
+    opts.frame_opts.snip_edges = False
+    opts.frame_opts.samp_freq = sample_rate
+    opts.mel_opts.num_bins = 80
+
+    online_fbank = knf.OnlineFbank(opts)
+    online_fbank.accept_waveform(sample_rate, (samples * 32768).tolist())
+    online_fbank.input_finished()
+
+    features = np.stack(
+        [online_fbank.get_frame(i) for i in range(online_fbank.num_frames_ready)]
+    )
+    assert features.data.contiguous is True
+    assert features.dtype == np.float32, features.dtype
+    mean = features.mean(axis=0, keepdims=True)
+    stddev = features.std(axis=0, keepdims=True)
+    features = (features - mean) / (stddev + 1e-5)
+    return features
+
+
+def load_tokens():
+    ans = dict()
+    with open("tokens.txt", encoding="utf-8") as f:
+        for line in f:
+            sym, idx = line.strip().split()
+            ans[int(idx)] = sym
+    return ans
+
+
+def main():
+    filename = "./test_wavs/0.wav"
+    features = compute_feat(filename)  # (T, C)
+    features = np.expand_dims(features, axis=0)  # (N, T, C)
+    features = features.transpose(0, 2, 1)  # (N, C, T)
+    print(features.shape)  # (N, C, T), (1, 80, 663)
+    features_length = np.array([features.shape[2]], dtype=np.int64)
+    print(features_length)
+
+    sess = ort.InferenceSession("model.onnx")
+
+    for n in sess.get_inputs():
+        print(n.name, n.type, n.shape)
+
+    for n in sess.get_outputs():
+        print(n.name, n.type, n.shape)
+
+    inputs = {
+        sess.get_inputs()[0].name: features,
+        sess.get_inputs()[1].name: features_length,
+    }
+
+    outputs = sess.run([sess.get_outputs()[0].name], input_feed=inputs)
+    # outputs[0] contains log_probs
+
+    print(outputs[0].shape)  # (N, T, C), (1, 166, 1025)
+    print(outputs[0].dtype)  # float32
+    print(np.exp(outputs[0]).sum(axis=-1).reshape(-1)[:10])  # validate it is log_probs
+    indexes = outputs[0].argmax(axis=-1)
+    print(indexes.shape)
+    indexes = indexes.squeeze().tolist()
+    unique_indexes = [k for k, _ in itertools.groupby(indexes)]
+    print(indexes)
+    print(unique_indexes)
+
+    tokens = load_tokens()
+    text = "".join([tokens[i] for i in unique_indexes if i != len(tokens) - 1])
+    print(text)
+
+    """
+audio_signal tensor(float) ['audio_signal_dynamic_axes_1', 80, 'audio_signal_dynamic_axes_2']
+length tensor(int64) ['length_dynamic_axes_1']
+logprobs tensor(float) ['logprobs_dynamic_axes_1', 'logprobs_dynamic_axes_2', 1025]
+    """
+
+
+if __name__ == "__main__":
+    main()

en/sherpa-onnx-nemo-ctc-en-conformer-large/test_wavs/0.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6bc58a4efdf20daac252b6b1502632601a71efe0308f6757dc1eda34891a7e4f
+size 212044

en/sherpa-onnx-nemo-ctc-en-conformer-large/test_wavs/1.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5143a6ba93c4b274e2c4ac22deb75c2c48936c853f0519add1de828b6c79cc5a
+size 534924

en/sherpa-onnx-nemo-ctc-en-conformer-large/test_wavs/trans.txt

@@ -0,0 +1,3 @@
+0.wav AFTER EARLY NIGHTFALL THE YELLOW LAMPS WOULD LIGHT UP HERE AND THERE THE SQUALID QUARTER OF THE BROTHELS
+1.wav GOD AS A DIRECT CONSEQUENCE OF THE SIN WHICH MAN THUS PUNISHED HAD GIVEN HER A LOVELY CHILD WHOSE PLACE WAS ON THAT SAME DISHONOURED BOSOM TO CONNECT HER PARENT FOR EVER WITH THE RACE AND DESCENT OF MORTALS AND TO BE FINALLY A BLESSED SOUL IN HEAVEN
+8k.wav YET THESE THOUGHTS AFFECTED HESTER PRYNNE LESS WITH HOPE THAN APPREHENSION

en/sherpa-onnx-nemo-ctc-en-conformer-large/tokens.txt

@@ -0,0 +1,129 @@
+<unk> 0
+▁ 1
+s 2
+t 3
+e 4
+d 5
+o 6
+▁the 7
+a 8
+i 9
+▁a 10
+u 11
+y 12
+m 13
+l 14
+n 15
+p 16
+re 17
+c 18
+h 19
+r 20
+▁s 21
+g 22
+▁to 23
+er 24
+ing 25
+f 26
+▁and 27
+an 28
+▁i 29
+k 30
+▁that 31
+' 32
+▁of 33
+▁in 34
+w 35
+▁p 36
+ed 37
+or 38
+al 39
+ar 40
+▁f 41
+en 42
+in 43
+b 44
+▁you 45
+▁w 46
+▁b 47
+le 48
+ll 49
+es 50
+▁it 51
+ve 52
+ur 53
+▁we 54
+▁re 55
+▁be 56
+ly 57
+▁is 58
+▁he 59
+▁o 60
+▁c 61
+it 62
+▁n 63
+▁on 64
+un 65
+▁t 66
+on 67
+se 68
+th 69
+ce 70
+▁do 71
+ic 72
+▁for 73
+▁th 74
+ion 75
+ch 76
+▁was 77
+ri 78
+ent 79
+▁g 80
+ver 81
+▁co 82
+li 83
+▁ha 84
+▁ma 85
+la 86
+ro 87
+v 88
+us 89
+▁ca 90
+▁di 91
+▁this 92
+ra 93
+▁st 94
+▁e 95
+▁not 96
+▁so 97
+▁de 98
+▁have 99
+ter 100
+ir 101
+▁go 102
+ation 103
+▁with 104
+ate 105
+▁me 106
+▁mo 107
+ment 108
+▁con 109
+▁but 110
+vi 111
+▁pro 112
+▁ho 113
+j 114
+▁com 115
+ight 116
+▁know 117
+▁what 118
+ect 119
+▁ex 120
+▁some 121
+▁would 122
+▁like 123
+x 124
+▁his 125
+q 126
+z 127
+<blk> 128