hyperbranchformer_13M.yaml

# ############################################################################
# Model: E2E ASR with Transformer
# Encoder: HyperConformer Encoder
# Decoder: Transformer Decoder + (CTC/ATT joint) beamsearch + TransformerLM
# Tokens: unigram
# losses: CTC + KLdiv (Label Smoothing loss)
# Training: Librispeech 960h
# Authors:  Juan Pablo Zuluaga, Florian Mai, Titouan Parcollet
# ############################################################################
# Seed needs to be set at top of yaml, before objects with parameters are made

seed: 7775
__set_seed: !apply:torch.manual_seed [!ref <seed>]
output_folder: !ref results/hyperbranchformer_13M/<seed>
output_wer_folder: !ref <output_folder>/
save_folder: !ref <output_folder>/save
train_log: !ref <output_folder>/train_log.txt

# Language model (LM) pretraining
# NB: To avoid mismatch, the speech recognizer must be trained with the same
# tokenizer used for LM training. Here, we download everything from the
# speechbrain HuggingFace repository. However, a local path pointing to a
# directory containing the lm.ckpt and tokenizer.ckpt may also be specified
# instead. E.g if you want to use your own LM / tokenizer.
pretrained_lm_tokenizer_path: speechbrain/asr-transformer-transformerlm-librispeech

# Data files
data_folder: !PLACEHOLDER  # e.g., /path/to/LibriSpeech
# If RIRS_NOISES dir exists in /localscratch/xxx_corpus/RIRS_NOISES
# then data_folder_rirs should be /localscratch/xxx_corpus
# otherwise the dataset will automatically be downloaded
# data_folder_rirs: !ref <data_folder>
train_splits: ["train-clean-100", "train-clean-360", "train-other-500"]
dev_splits: ["dev-clean"]
test_splits: ["test-clean", "test-other"]
skip_prep: False
train_csv: !ref <output_folder>/train.csv
valid_csv: !ref <output_folder>/dev-clean.csv
test_csv:
    - !ref <output_folder>/test-clean.csv
    - !ref <output_folder>/test-other.csv

############################## Training Parameters #############################

# To make Transformers converge, the global bath size should be large enough.
# The global batch size is computed as batch_size * n_gpus * grad_accumulation_factor.
# Empirically, we found that this value should be >= 128.
# Please, set your parameters accordingly.
number_of_epochs: 110
batch_size: 16 # This works for 2x GPUs with 32GB
ctc_weight: 0.3
grad_accumulation_factor: 3
max_grad_norm: 5.0
loss_reduction: 'batchmean'
sorting: random
num_workers: 4
avg_checkpoints: 10 # Number of checkpoints to average for evaluation

# stages related parameters
lr_adam: 0.001

# Feature parameters
sample_rate: 16000
n_fft: 400
n_mels: 80

# This setup works well for a P40 24GB GPU, adapt it to your needs.
# Or turn it off (but training speed will decrease)
dynamic_batching: True
max_batch_length_train: 600
max_batch_length_val: 100 # we reduce it as the beam is much wider (VRAM)
num_bucket: 200
shuffle: True # if true re-creates batches at each epoch shuffling examples.
max_batch_ex: 128
batch_ordering: random

dynamic_batch_sampler_train:
    max_batch_length: !ref <max_batch_length_train>
    num_buckets: !ref <num_bucket>
    shuffle: !ref <shuffle>
    batch_ordering: !ref <batch_ordering>
    max_batch_ex: !ref <max_batch_ex>

dynamic_batch_sampler_valid:
    max_batch_length: !ref <max_batch_length_val>
    num_buckets: !ref <num_bucket>
    shuffle: !ref <shuffle>
    batch_ordering: !ref <batch_ordering>
    max_batch_ex: !ref <max_batch_ex>

# Dataloader options
train_dataloader_opts:
    batch_size: !ref <batch_size>
    shuffle: True
    num_workers: !ref <num_workers>

valid_dataloader_opts:
    batch_size: 1

test_dataloader_opts:
    batch_size: 1

####################### Model Parameters #######################################

# Transformer
d_model: 144
nhead: 8
num_encoder_layers: 10
num_decoder_layers: 4
csgu_linear_units: 3072
csgu_kernel_size: 31
transformer_dropout: 0.1
activation: !name:torch.nn.GELU
output_neurons: 5000
# specify 'hypermixing' for usage of multi-head HyperMixer instead of MultiHeadAttention
# You can also specify RelPosMHAXL for conformer
attention_type: hypermixing

# option 1) 'conformer' for HyperConformer; option 2) 'transformer' for vanilla HyperMixer
encoder_module: branchformer

# Outputs
blank_index: 0
label_smoothing: 0.1
pad_index: 0
bos_index: 1
eos_index: 2

# Decoding parameters
min_decode_ratio: 0.0
max_decode_ratio: 1.0
valid_search_interval: 30
valid_beam_size: 10
test_beam_size: 66
lm_weight: 0.60
ctc_weight_decode: 0.40

############################## Models ##########################################

CNN: !new:speechbrain.lobes.models.convolution.ConvolutionFrontEnd
    input_shape: (8, 10, 80)
    num_blocks: 2
    num_layers_per_block: 1
    out_channels: (64, 32)
    kernel_sizes: (3, 3)
    strides: (2, 2)
    residuals: (False, False)

Transformer: !new:speechbrain.lobes.models.transformer.TransformerASR.TransformerASR # yamllint disable-line rule:line-length
    input_size: 640
    tgt_vocab: !ref <output_neurons>
    d_model: !ref <d_model>
    nhead: !ref <nhead>
    num_encoder_layers: !ref <num_encoder_layers>
    num_decoder_layers: !ref <num_decoder_layers>
    dropout: !ref <transformer_dropout>
    activation: !ref <activation>
    branchformer_activation: !ref <activation>
    encoder_module: !ref <encoder_module>
    csgu_linear_units: !ref <csgu_linear_units>
    kernel_size: !ref <csgu_kernel_size>
    attention_type: !ref <attention_type>
    normalize_before: True
    causal: False

# This is the TransformerLM that is used according to the Huggingface repository
# Visit the HuggingFace model corresponding to the pretrained_lm_tokenizer_path
# For more details about the model!
# NB: It has to match the pre-trained TransformerLM!!
lm_model: !new:speechbrain.lobes.models.transformer.TransformerLM.TransformerLM # yamllint disable-line rule:line-length
    vocab: !ref <output_neurons>
    d_model: 768
    nhead: 12
    num_encoder_layers: 12
    num_decoder_layers: 0
    d_ffn: 3072
    dropout: 0.0
    activation: !name:torch.nn.GELU
    normalize_before: False

tokenizer: !new:sentencepiece.SentencePieceProcessor

ctc_lin: !new:speechbrain.nnet.linear.Linear
    input_size: !ref <d_model>
    n_neurons: !ref <output_neurons>

seq_lin: !new:speechbrain.nnet.linear.Linear
    input_size: !ref <d_model>
    n_neurons: !ref <output_neurons>

normalize: !new:speechbrain.processing.features.InputNormalization
    norm_type: global
    update_until_epoch: 4

modules:
    CNN: !ref <CNN>
    Transformer: !ref <Transformer>
    seq_lin: !ref <seq_lin>
    ctc_lin: !ref <ctc_lin>
    normalize: !ref <normalize>

model: !new:torch.nn.ModuleList
    - [!ref <CNN>, !ref <Transformer>, !ref <seq_lin>, !ref <ctc_lin>]

# define two optimizers here for two-stage training
Adam: !name:torch.optim.Adam
    lr: !ref <lr_adam>
    betas: (0.9, 0.98)
    eps: 0.000000001

############################## Decoding & optimiser ############################

ctc_scorer: !new:speechbrain.decoders.scorer.CTCScorer
    eos_index: !ref <eos_index>
    blank_index: !ref <blank_index>
    ctc_fc: !ref <ctc_lin>

transformerlm_scorer: !new:speechbrain.decoders.scorer.TransformerLMScorer
    language_model: !ref <lm_model>
    temperature: 1.15

scorer_valid_search: !new:speechbrain.decoders.scorer.ScorerBuilder
    full_scorers: [!ref <ctc_scorer>]
    weights:
        ctc: !ref <ctc_weight_decode>

scorer_test_search: !new:speechbrain.decoders.scorer.ScorerBuilder
    full_scorers: [!ref <transformerlm_scorer>, !ref <ctc_scorer>]
    weights:
        ctc: !ref <ctc_weight_decode>
        transformerlm: !ref <lm_weight>

valid_search: !new:speechbrain.decoders.S2STransformerBeamSearcher
    modules: [!ref <Transformer>, !ref <seq_lin>]
    bos_index: !ref <bos_index>
    eos_index: !ref <eos_index>
    min_decode_ratio: !ref <min_decode_ratio>
    max_decode_ratio: !ref <max_decode_ratio>
    beam_size: !ref <valid_beam_size>
    using_eos_threshold: False
    length_normalization: True
    scorer: !ref <scorer_valid_search>

test_search: !new:speechbrain.decoders.S2STransformerBeamSearcher
    modules: [!ref <Transformer>, !ref <seq_lin>]
    bos_index: !ref <bos_index>
    eos_index: !ref <eos_index>
    min_decode_ratio: !ref <min_decode_ratio>
    max_decode_ratio: !ref <max_decode_ratio>
    beam_size: !ref <test_beam_size>
    temperature: 1.15
    using_eos_threshold: False
    length_normalization: True
    scorer: !ref <scorer_test_search>

log_softmax: !new:torch.nn.LogSoftmax
    dim: -1

ctc_cost: !name:speechbrain.nnet.losses.ctc_loss
    blank_index: !ref <blank_index>
    reduction: !ref <loss_reduction>

seq_cost: !name:speechbrain.nnet.losses.kldiv_loss
    label_smoothing: !ref <label_smoothing>
    reduction: !ref <loss_reduction>

noam_annealing: !new:speechbrain.nnet.schedulers.NoamScheduler
    lr_initial: !ref <lr_adam>
    n_warmup_steps: 25000

checkpointer: !new:speechbrain.utils.checkpoints.Checkpointer
    checkpoints_dir: !ref <save_folder>
    recoverables:
        model: !ref <model>
        noam_scheduler: !ref <noam_annealing>
        normalizer: !ref <normalize>
        counter: !ref <epoch_counter>

epoch_counter: !new:speechbrain.utils.epoch_loop.EpochCounter
    limit: !ref <number_of_epochs>

############################## Augmentations ###################################

# Speed perturbation
speed_perturb: !new:speechbrain.augment.time_domain.SpeedPerturb
    orig_freq: !ref <sample_rate>
    speeds: [95, 100, 105]

# Time Drop
time_drop: !new:speechbrain.augment.freq_domain.SpectrogramDrop
    drop_length_low: 15
    drop_length_high: 25
    drop_count_low: 4
    drop_count_high: 4
    replace: "mean"

# Freq Drop
freq_drop: !new:speechbrain.augment.freq_domain.SpectrogramDrop
    drop_length_low: 10
    drop_length_high: 20
    drop_count_low: 4
    drop_count_high: 4
    replace: "mean"
    dim: 2

# Time warp
time_warp: !new:speechbrain.augment.freq_domain.Warping

fea_augment: !new:speechbrain.augment.augmenter.Augmenter
    min_augmentations: 3
    max_augmentations: 3
    augment_prob: 1.0
    augmentations: [
        !ref <time_drop>,
        !ref <freq_drop>,
        !ref <time_warp>]

compute_features: !new:speechbrain.lobes.features.Fbank
    sample_rate: !ref <sample_rate>
    n_fft: !ref <n_fft>
    n_mels: !ref <n_mels>

############################## Logging and Pretrainer ##########################

train_logger: !new:speechbrain.utils.train_logger.FileTrainLogger
    save_file: !ref <train_log>

error_rate_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats
acc_computer: !name:speechbrain.utils.Accuracy.AccuracyStats

# The pretrainer allows a mapping between pretrained files and instances that
# are declared in the yaml. E.g here, we will download the file lm.ckpt
# and it will be loaded into "lm" which is pointing to the <lm_model> defined
# before.
pretrainer: !new:speechbrain.utils.parameter_transfer.Pretrainer
    collect_in: !ref <save_folder>
    loadables:
        lm: !ref <lm_model>
        tokenizer: !ref <tokenizer>
    paths:
        lm: !ref <pretrained_lm_tokenizer_path>/lm.ckpt
        tokenizer: !ref <pretrained_lm_tokenizer_path>/tokenizer.ckpt