train_ASR_transformer.yaml

# ############################################################################
# Model: E2E ASR with Transformer
# Encoder: Transformer Encoder
# Decoder: Transformer Decoder + (CTC/ATT joint) beamsearch
# Tokens: BPE with unigram
# losses: CTC + KLdiv (Label Smoothing loss)
# Training: AISHELL-1
# Authors:  Jianyuan Zhong, Titouan Parcollet
# ############################################################################
# Seed needs to be set at top of yaml, before objects with parameters are made
seed: 8886
__set_seed: !apply:torch.manual_seed [!ref <seed>]
output_folder: !ref results/transformer/<seed>
cer_file: !ref <output_folder>/cer.txt
save_folder: !ref <output_folder>/save
train_log: !ref <output_folder>/train_log.txt

# Data for augmentation
NOISE_DATASET_URL: https://www.dropbox.com/scl/fi/a09pj97s5ifan81dqhi4n/noises.zip?rlkey=j8b0n9kdjdr32o1f06t0cw5b7&dl=1

# Data files
data_folder: !PLACEHOLDER # e,g./path/to/aishell
data_folder_noise: !ref <data_folder>/noise # The noisy sequences for data augmentation will automatically be downloaded here.
skip_prep: False
remove_compressed_wavs: False
ckpt_interval_minutes: 15 # save checkpoint every N min
train_data: !ref <save_folder>/train.csv
valid_data: !ref <save_folder>/dev.csv
test_data: !ref <save_folder>/test.csv
noise_annotation: !ref <save_folder>/noise.csv #The data manifest files are created by the data preparation script
tokenizer_file: speechbrain/asr-transformer-aishell/tokenizer.ckpt

####################### Training Parameters ####################################

number_of_epochs: 50
batch_size: 8
ctc_weight: 0.3
grad_accumulation_factor: 4
loss_reduction: 'batchmean'
sorting: random
avg_checkpoints: 10 # Number of checkpoints to average for evaluation
precision: fp32 # bf16, fp16 or fp32

dynamic_batching: False
max_batch_length: 15 # in terms of "duration" in annotations by default, second here
shuffle: False # if true re-creates batches at each epoch shuffling examples.
num_buckets: 10 # floor(log(max_batch_len/left_bucket_len, multiplier)) + 1
batch_ordering: ascending
dynamic_batch_sampler:
    max_batch_length: !ref <max_batch_length>
    shuffle: !ref <shuffle>
    num_buckets: !ref <num_buckets>
    batch_ordering: !ref <batch_ordering>

num_workers: 4

# stages related parameters
stage_one_epochs: 40
lr_adam: 1.0
lr_sgd: 0.000025

# Feature parameters
sample_rate: 16000
n_fft: 400
n_mels: 80

# Dataloader options
train_dataloader_opts:
    batch_size: !ref <batch_size>
    num_workers: !ref <num_workers>
    shuffle: True

valid_dataloader_opts:
    batch_size: !ref <batch_size>
    num_workers: !ref <num_workers>

test_dataloader_opts:
    batch_size: !ref <batch_size>
    num_workers: !ref <num_workers>

####################### Model Parameters #######################################
# Transformer
d_model: 256
nhead: 4
num_encoder_layers: 12
num_decoder_layers: 6
d_ffn: 2048
transformer_dropout: 0.1
activation: !name:torch.nn.GELU
output_neurons: 5000

# Outputs
blank_index: 0
label_smoothing: 0.1
pad_index: 0
bos_index: 1
eos_index: 2

# Decoding parameters
min_decode_ratio: 0.0
max_decode_ratio: 1.0 # 1.0
valid_search_interval: 10
valid_beam_size: 10
test_beam_size: 10
ctc_weight_decode: 0.40

############################## Models ##########################################

CNN: !new:speechbrain.lobes.models.convolution.ConvolutionFrontEnd
    input_shape: (8, 10, 80)
    num_blocks: 2
    num_layers_per_block: 1
    out_channels: (256, 256)
    kernel_sizes: (3, 3)
    strides: (2, 2)
    residuals: (False, False)

Transformer: !new:speechbrain.lobes.models.transformer.TransformerASR.TransformerASR # yamllint disable-line rule:line-length
    input_size: 5120
    tgt_vocab: !ref <output_neurons>
    d_model: !ref <d_model>
    nhead: !ref <nhead>
    num_encoder_layers: !ref <num_encoder_layers>
    num_decoder_layers: !ref <num_decoder_layers>
    d_ffn: !ref <d_ffn>
    dropout: !ref <transformer_dropout>
    activation: !ref <activation>
    normalize_before: True
    causal: False

tokenizer: !new:sentencepiece.SentencePieceProcessor

ctc_lin: !new:speechbrain.nnet.linear.Linear
    input_size: !ref <d_model>
    n_neurons: !ref <output_neurons>

seq_lin: !new:speechbrain.nnet.linear.Linear
    input_size: !ref <d_model>
    n_neurons: !ref <output_neurons>


modules:
    CNN: !ref <CNN>
    Transformer: !ref <Transformer>
    seq_lin: !ref <seq_lin>
    ctc_lin: !ref <ctc_lin>

model: !new:torch.nn.ModuleList
    - [!ref <CNN>, !ref <Transformer>, !ref <seq_lin>, !ref <ctc_lin>]

# define two optimizers here for two-stage training
Adam: !name:torch.optim.Adam
    lr: 0
    betas: (0.9, 0.98)
    eps: 0.000000001

SGD: !name:torch.optim.SGD
    lr: !ref <lr_sgd>
    momentum: 0.99
    nesterov: True

############################## Decoding & optimiser ############################

ctc_scorer: !new:speechbrain.decoders.scorer.CTCScorer
    eos_index: !ref <eos_index>
    blank_index: !ref <blank_index>
    ctc_fc: !ref <ctc_lin>

scorer: !new:speechbrain.decoders.scorer.ScorerBuilder
    full_scorers: [!ref <ctc_scorer>]
    weights:
        ctc: !ref <ctc_weight_decode>

valid_search: !new:speechbrain.decoders.S2STransformerBeamSearcher
    modules: [!ref <Transformer>, !ref <seq_lin>]
    bos_index: !ref <bos_index>
    eos_index: !ref <eos_index>
    min_decode_ratio: !ref <min_decode_ratio>
    max_decode_ratio: !ref <max_decode_ratio>
    beam_size: !ref <valid_beam_size>
    using_eos_threshold: False
    length_normalization: True
    scorer: !ref <scorer>

test_search: !new:speechbrain.decoders.S2STransformerBeamSearcher
    modules: [!ref <Transformer>, !ref <seq_lin>]
    bos_index: !ref <bos_index>
    eos_index: !ref <eos_index>
    min_decode_ratio: !ref <min_decode_ratio>
    max_decode_ratio: !ref <max_decode_ratio>
    beam_size: !ref <test_beam_size>
    using_eos_threshold: False
    length_normalization: True
    scorer: !ref <scorer>

log_softmax: !new:torch.nn.LogSoftmax
    dim: -1

ctc_cost: !name:speechbrain.nnet.losses.ctc_loss
    blank_index: !ref <blank_index>
    reduction: !ref <loss_reduction>

seq_cost: !name:speechbrain.nnet.losses.kldiv_loss
    label_smoothing: !ref <label_smoothing>
    reduction: !ref <loss_reduction>

noam_annealing: !new:speechbrain.nnet.schedulers.NoamScheduler
    lr_initial: !ref <lr_adam>
    n_warmup_steps: 25000
    model_size: !ref <d_model>

checkpointer: !new:speechbrain.utils.checkpoints.Checkpointer
    checkpoints_dir: !ref <save_folder>
    recoverables:
        model: !ref <model>
        noam_scheduler: !ref <noam_annealing>
        normalizer: !ref <normalize>
        counter: !ref <epoch_counter>

epoch_counter: !new:speechbrain.utils.epoch_loop.EpochCounter
    limit: !ref <number_of_epochs>

normalize: !new:speechbrain.processing.features.InputNormalization
    norm_type: global
    update_until_epoch: 4


compute_features: !new:speechbrain.lobes.features.Fbank
    sample_rate: !ref <sample_rate>
    n_fft: !ref <n_fft>
    n_mels: !ref <n_mels>

############################## Augmentation ####################################

# Download and prepare the dataset of noisy sequences for augmentation
prepare_noise_data: !name:speechbrain.augment.preparation.prepare_dataset_from_URL
    URL: !ref <NOISE_DATASET_URL>
    dest_folder: !ref <data_folder_noise>
    ext: wav
    csv_file: !ref <noise_annotation>


# Add noise to input signal
add_noise: !new:speechbrain.augment.time_domain.AddNoise
    csv_file: !ref <noise_annotation>
    snr_low: 0
    snr_high: 15
    noise_sample_rate: !ref <sample_rate>
    clean_sample_rate: !ref <sample_rate>
    num_workers: !ref <num_workers>

# Augmenter: Combines previously defined augmentations to perform data augmentation
wav_augment: !new:speechbrain.augment.augmenter.Augmenter
    concat_original: True
    min_augmentations: 1
    max_augmentations: 1
    augment_prob: 1.0
    augmentations: [
        !ref <add_noise>]

# Time Drop
time_drop: !new:speechbrain.augment.freq_domain.SpectrogramDrop
    drop_length_low: 0
    drop_length_high: 100
    drop_count_low: 2
    drop_count_high: 2

# Frequency Drop
freq_drop: !new:speechbrain.augment.freq_domain.SpectrogramDrop
    drop_length_low: 30
    drop_length_high: 40
    drop_count_low: 2
    drop_count_high: 2
    dim: 2

# Time warp
time_warp: !new:speechbrain.augment.freq_domain.Warping

fea_augment: !new:speechbrain.augment.augmenter.Augmenter
    concat_original: True
    min_augmentations: 1
    max_augmentations: 1
    augment_start_index: !ref <batch_size> # This leaves original inputs unchanged
    concat_end_index: !ref <batch_size> # This leaves original inputs unchanged
    augment_prob: 1.0
    augmentations: [
        !ref <time_drop>,
        !ref <freq_drop>,
        !ref <time_warp>]

############################## Logging and Pretrainer ##########################

train_logger: !new:speechbrain.utils.train_logger.FileTrainLogger
    save_file: !ref <train_log>

# AISHELL-1 has spaces between words in the transcripts,
# which Chinese writing normally does not do.
# If remove_spaces, spaces are removed
# from the transcript before computing CER.
remove_spaces: True
split_tokens: !apply:operator.not_ [!ref <remove_spaces>]

cer_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats
    split_tokens: !ref <split_tokens>
acc_computer: !name:speechbrain.utils.Accuracy.AccuracyStats

pretrainer: !new:speechbrain.utils.parameter_transfer.Pretrainer
    collect_in: !ref <save_folder>
    loadables:
        tokenizer: !ref <tokenizer>
    paths:
        tokenizer: !ref <tokenizer_file>