diff --git a/datasets/kws20.py b/datasets/kws20.py
index e63cc1c24..245450ede 100644
--- a/datasets/kws20.py
+++ b/datasets/kws20.py
@@ -1,6 +1,7 @@
#
# Copyright (c) 2018 Intel Corporation
# Portions Copyright (C) 2019-2023 Maxim Integrated Products, Inc.
+# Portions Copyright (C) 2023-2024 Analog Devices, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -44,14 +45,14 @@ class KWS:
Dataset, 1D folded.
Args:
- root (string): Root directory of dataset where ``KWS/processed/dataset.pt``
- exist.
+ root (string): Root directory of dataset where ``KWS/processed/dataset.pt`` exist.
classes(array): List of keywords to be used.
d_type(string): Option for the created dataset. ``train`` or ``test``.
- n_augment(int, optional): Number of augmented samples added to the dataset from
- each sample by random modifications, i.e. stretching, shifting and random noise.
- transform (callable, optional): A function/transform that takes in an PIL image
- and returns a transformed version.
+ transform (callable, optional): A function/transform that takes in a signal between [0, 1]
+ and returns a transformed version, suitable for ai8x training / evaluation.
+ quantization_scheme (dict, optional): Dictionary containing quantization scheme parameters.
+ If not provided, default values are used.
+ augmentation (dict, optional): Dictionary containing augmentation parameters.
download (bool, optional): If true, downloads the dataset from the internet and
puts it in root directory. If dataset is already downloaded, it is not
downloaded again.
@@ -86,15 +87,15 @@ def __init__(self, root, classes, d_type, t_type, transform=None, quantization_s
self.__parse_augmentation(augmentation)
if not self.save_unquantized:
- self.data_file = 'dataset2.pt'
+ self.data_file = 'dataset.pt'
else:
self.data_file = 'unquantized.pt'
if download:
self.__download()
- self.data, self.targets, self.data_type = torch.load(os.path.join(
- self.processed_folder, self.data_file))
+ self.data, self.targets, self.data_type, self.shift_limits = \
+ torch.load(os.path.join(self.processed_folder, self.data_file))
print(f'\nProcessing {self.d_type}...')
self.__filter_dtype()
@@ -156,10 +157,6 @@ def __parse_augmentation(self, augmentation):
print('No key `shift` in input augmentation dictionary! '
'Using defaults: [Min:-0.1, Max: 0.1]')
self.augmentation['shift'] = {'min': -0.1, 'max': 0.1}
- if 'stretch' not in augmentation:
- print('No key `stretch` in input augmentation dictionary! '
- 'Using defaults: [Min: 0.8, Max: 1.3]')
- self.augmentation['stretch'] = {'min': 0.8, 'max': 1.3}
def __download(self):
@@ -363,23 +360,27 @@ def __filter_dtype(self):
# take a copy of the original data and targets temporarily for validation set
self.data_original = self.data.clone()
self.targets_original = self.targets.clone()
+ self.data_type_original = self.data_type.clone()
self.data = self.data[idx_to_select, :]
self.targets = self.targets[idx_to_select, :]
+ self.data_type = self.data_type[idx_to_select, :]
# append validation set to the training set if validation examples are explicitly included
if self.d_type == 'train':
- idx_to_select = (self.data_type == 2)[:, -1]
+ idx_to_select = (self.data_type_original == 2)[:, -1]
if idx_to_select.sum() > 0: # if validation examples exist
self.data = torch.cat((self.data, self.data_original[idx_to_select, :]), dim=0)
self.targets = \
torch.cat((self.targets, self.targets_original[idx_to_select, :]), dim=0)
+ self.data_type = \
+ torch.cat((self.data_type, self.data_type_original[idx_to_select, :]), dim=0)
# indicate the list of validation indices to be used by distiller's dataloader
self.valid_indices = range(set_size, set_size + idx_to_select.sum())
print(f'validation set: {idx_to_select.sum()} elements')
del self.data_original
del self.targets_original
- del self.data_type
+ del self.data_type_original
def __filter_classes(self):
initial_new_class_label = len(self.class_dict)
@@ -401,28 +402,63 @@ def __filter_classes(self):
def __len__(self):
return len(self.data)
+ def __reshape_audio(self, audio, row_len=128):
+ # add overlap if necessary later on
+ return torch.transpose(audio.reshape((-1, row_len)), 1, 0)
+
+ def shift_and_noise_augment(self, audio, shift_limits):
+ """Augments audio by adding random shift and noise.
+ """
+ random_noise_var_coeff = np.random.uniform(self.augmentation['noise_var']['min'],
+ self.augmentation['noise_var']['max'])
+ random_shift_sample = np.random.randint(shift_limits[0], shift_limits[1])
+
+ aug_audio = self.shift(audio, random_shift_sample)
+ aug_audio = self.add_quantized_white_noise(aug_audio, random_noise_var_coeff)
+
+ return aug_audio
+
def __getitem__(self, index):
- inp, target = self.data[index].type(torch.FloatTensor), int(self.targets[index])
+ inp, target = self.data[index], int(self.targets[index])
+ data_type, shift_limits = self.data_type[index], self.shift_limits[index]
+
+ # apply dynamic shift and noise augmentation to training examples
+ if data_type == 0:
+ inp = self.shift_and_noise_augment(inp, shift_limits)
+
+ # reshape to 2D
+ inp = self.__reshape_audio(inp)
+
+ inp = inp.type(torch.FloatTensor)
+
if not self.save_unquantized:
inp /= 256
if self.transform is not None:
inp = self.transform(inp)
+
return inp, target
@staticmethod
def add_white_noise(audio, noise_var_coeff):
- """Adds zero mean Gaussian noise to image with specified variance.
+ """Adds zero mean Gaussian noise to the audio with specified variance.
"""
coeff = noise_var_coeff * np.mean(np.abs(audio))
noisy_audio = audio + coeff * np.random.randn(len(audio))
return noisy_audio
@staticmethod
- def shift(audio, shift_sec, fs):
+ def add_quantized_white_noise(audio, noise_var_coeff):
+ """Adds zero mean Gaussian noise to the audio with specified variance.
+ """
+ coeff = noise_var_coeff * torch.mean(torch.abs(audio.type(torch.float)-128))
+ noise = (coeff * torch.randn(len(audio))).type(torch.int16)
+ return (audio + noise).clip(0, 255).type(torch.uint8)
+
+ @staticmethod
+ def shift(audio, shift_sample):
"""Shifts audio.
"""
- shift_count = int(shift_sec * fs)
- return np.roll(audio, shift_count)
+ return torch.roll(audio, shift_sample)
@staticmethod
def stretch(audio, rate=1):
@@ -437,36 +473,6 @@ def stretch(audio, rate=1):
return audio2
- def augment(self, audio, fs, verbose=False):
- """Augments audio by adding random noise, shift and stretch ratio.
- """
- random_noise_var_coeff = np.random.uniform(self.augmentation['noise_var']['min'],
- self.augmentation['noise_var']['max'])
- random_shift_time = np.random.uniform(self.augmentation['shift']['min'],
- self.augmentation['shift']['max'])
- random_stretch_coeff = np.random.uniform(self.augmentation['stretch']['min'],
- self.augmentation['stretch']['max'])
-
- sox_effects = [["speed", str(random_stretch_coeff)], ["rate", str(fs)]]
- aug_audio, _ = torchaudio.sox_effects.apply_effects_tensor(
- torch.unsqueeze(torch.from_numpy(audio).float(), dim=0), fs, sox_effects)
- aug_audio = aug_audio.numpy().squeeze()
- aug_audio = self.shift(aug_audio, random_shift_time, fs)
- aug_audio = self.add_white_noise(aug_audio, random_noise_var_coeff)
-
- if verbose:
- print(f'random_noise_var_coeff: {random_noise_var_coeff:.2f}\nrandom_shift_time: \
- {random_shift_time:.2f}\nrandom_stretch_coeff: {random_stretch_coeff:.2f}')
- return aug_audio
-
- def augment_multiple(self, audio, fs, n_augment, verbose=False):
- """Calls `augment` function for n_augment times for given audio data.
- Finally the original audio is added to have (n_augment+1) audio data.
- """
- aug_audio = [self.augment(audio, fs, verbose=verbose) for i in range(n_augment)]
- aug_audio.insert(0, audio)
- return aug_audio
-
@staticmethod
def compand(data, mu=255):
"""Compand the signal level to warp from Laplacian distribution to uniform distribution"""
@@ -498,9 +504,59 @@ def quantize_audio(data, num_bits=8, compand=False, mu=255):
q_data = np.clip(q_data, 0, max_val)
return np.uint8(q_data)
- def __gen_datasets(self, exp_len=16384, row_len=128, overlap_ratio=0):
- print('Generating dataset from raw data samples for the first time. ')
- print('This process will take significant time (~60 minutes)...')
+ def get_audio_endpoints(self, audio, fs):
+ """Future: May implement a method to detect the beginning & end of voice activity in audio.
+ Currently, it returns end points compatible with augmentation['shift'] values
+ """
+ return int(-self.augmentation['shift']['min'] * fs), \
+ int(len(audio) - self.augmentation['shift']['max'] * fs)
+
+ def speed_augment(self, audio, fs, sample_no=0):
+ """Augments audio by randomly changing the speed of the audio.
+ The generated coefficient follows 0.9, 1.1, 0.95, 1.05... pattern
+ """
+ speed_multiplier = 1.0 + 0.2 * (sample_no % 2 - 0.5) / (1 + sample_no // 2)
+
+ sox_effects = [["speed", str(speed_multiplier)], ["rate", str(fs)]]
+ aug_audio, _ = torchaudio.sox_effects.apply_effects_tensor(
+ torch.unsqueeze(torch.from_numpy(audio).float(), dim=0), fs, sox_effects)
+ aug_audio = aug_audio.numpy().squeeze()
+
+ return aug_audio, speed_multiplier
+
+ def speed_augment_multiple(self, audio, fs, exp_len, n_augment):
+ """Calls `speed_augment` function for n_augment times for given audio data.
+ Finally the original audio is added to have (n_augment+1) audio data.
+ """
+ aug_audio = [None] * (n_augment + 1)
+ aug_speed = np.ones((n_augment + 1,))
+ shift_limits = np.zeros((n_augment + 1, 2))
+ voice_begin_idx, voice_end_idx = self.get_audio_endpoints(audio, fs)
+ aug_audio[0] = audio
+ for i in range(n_augment):
+ aug_audio[i+1], aug_speed[i+1] = self.speed_augment(audio, fs, sample_no=i)
+ for i in range(n_augment + 1):
+ if len(aug_audio[i]) < exp_len:
+ aug_audio[i] = np.pad(aug_audio[i], (0, exp_len - len(aug_audio[i])), 'constant')
+ aug_begin_idx = voice_begin_idx * aug_speed[i]
+ aug_end_idx = voice_end_idx * aug_speed[i]
+ if aug_end_idx - aug_begin_idx <= exp_len:
+ # voice activity duration is shorter than the expected length
+ segment_begin = max(aug_end_idx, exp_len) - exp_len
+ segment_end = max(aug_end_idx, exp_len)
+ aug_audio[i] = aug_audio[i][segment_begin:segment_end]
+ shift_limits[i, 0] = -aug_begin_idx + (max(aug_end_idx, exp_len) - exp_len)
+ shift_limits[i, 1] = max(aug_end_idx, exp_len) - aug_end_idx
+ else:
+ # voice activity duraction is longer than the expected length
+ midpoint = (aug_begin_idx + aug_end_idx) // 2
+ aug_audio[i] = aug_audio[i][midpoint - exp_len // 2: midpoint + exp_len // 2]
+ shift_limits[i, :] = [0, 0]
+ return aug_audio, shift_limits
+
+ def __gen_datasets(self, exp_len=16384):
+ print('Generating dataset from raw data samples for the first time.')
+ print('This process may take a few minutes.')
with warnings.catch_warnings():
warnings.simplefilter('error')
@@ -508,12 +564,6 @@ def __gen_datasets(self, exp_len=16384, row_len=128, overlap_ratio=0):
labels = [d for d in lst if os.path.isdir(os.path.join(self.raw_folder, d))
and d[0].isalpha()]
- # PARAMETERS
- overlap = int(np.ceil(row_len * overlap_ratio))
- num_rows = int(np.ceil(exp_len / (row_len - overlap)))
- data_len = int((num_rows * row_len - (num_rows - 1) * overlap))
- print(f'data_len: {data_len}')
-
# show the size of dataset for each keyword
print('------------- Label Size ---------------')
for i, label in enumerate(labels):
@@ -534,27 +584,36 @@ def __gen_datasets(self, exp_len=16384, row_len=128, overlap_ratio=0):
for i, label in enumerate(labels):
print(f'Processing the label: {label}. {i + 1} of {len(labels)}')
record_list = sorted(os.listdir(os.path.join(self.raw_folder, label)))
+ record_len = len(record_list)
+
+ # get the number testing samples for the class
+ test_count_class = 0
+ for r, record_name in enumerate(record_list):
+ local_filename = os.path.join(label, record_name)
+ if local_filename in testing_set:
+ test_count_class += 1
+
+ # no augmentation for testing set, subtract them accordingly
+ number_of_total_samples = record_len * (self.augmentation['aug_num'] + 1) - \
+ test_count_class * self.augmentation['aug_num']
- # dimension: row_length x number_of_rows
if not self.save_unquantized:
- data_in = np.empty(((self.augmentation['aug_num'] + 1) * len(record_list),
- row_len, num_rows), dtype=np.uint8)
+ data_in = np.empty((number_of_total_samples, exp_len), dtype=np.uint8)
else:
- data_in = np.empty(((self.augmentation['aug_num'] + 1) * len(record_list),
- row_len, num_rows), dtype=np.float32)
- data_type = np.empty(((self.augmentation['aug_num'] + 1) * len(record_list), 1),
- dtype=np.uint8)
- # create data classes
- data_class = np.full(((self.augmentation['aug_num'] + 1) * len(record_list), 1), i,
- dtype=np.uint8)
+ data_in = np.empty((number_of_total_samples, exp_len), dtype=np.float32)
+
+ data_type = np.empty((number_of_total_samples, 1), dtype=np.uint8)
+ data_shift_limits = np.empty((number_of_total_samples, 2), dtype=np.int16)
+ data_class = np.full((number_of_total_samples, 1), i, dtype=np.uint8)
time_s = time.time()
+ sample_index = 0
for r, record_name in enumerate(record_list):
local_filename = os.path.join(label, record_name)
if r % 1000 == 0:
- print(f'\t{r + 1} of {len(record_list)}')
+ print(f'\t{r + 1} of {record_len}')
if local_filename in testing_set:
d_typ = np.uint8(1) # test
@@ -568,29 +627,28 @@ def __gen_datasets(self, exp_len=16384, row_len=128, overlap_ratio=0):
record_pth = os.path.join(self.raw_folder, label, record_name)
record, fs = librosa.load(record_pth, offset=0, sr=None)
- audio_seq_list = self.augment_multiple(record, fs,
- self.augmentation['aug_num'])
- for n_a, audio_seq in enumerate(audio_seq_list):
- # store set type: train, test or validate
- data_type[(self.augmentation['aug_num'] + 1) * r + n_a, 0] = d_typ
-
- # Write audio 128x128=16384 samples without overlap
- for n_r in range(num_rows):
- start_idx = n_r * (row_len - overlap)
- end_idx = start_idx + row_len
- audio_chunk = audio_seq[start_idx:end_idx]
- # pad zero if the length of the chunk is smaller than row_len
- audio_chunk = np.pad(audio_chunk, [0, row_len - audio_chunk.size])
- # store input data after quantization
- data_idx = (self.augmentation['aug_num'] + 1) * r + n_a
- if not self.save_unquantized:
- data_in[data_idx, :, n_r] = \
- KWS.quantize_audio(audio_chunk,
- num_bits=self.quantization['bits'],
- compand=self.quantization['compand'],
- mu=self.quantization['mu'])
- else:
- data_in[data_idx, :, n_r] = audio_chunk
+
+ # normalize dynamic range to [-1, +1]
+ record = record / np.max(np.abs(record))
+
+ if d_typ != 1: # training and validation examples get speed augmentation
+ no_augmentations = self.augmentation['aug_num']
+ else: # test examples don't get speed augmentation
+ no_augmentations = 0
+
+ # apply speed augmentations and calculate shift limits
+ audio_seq_list, shift_limits = \
+ self.speed_augment_multiple(record, fs, exp_len, no_augmentations)
+
+ for local_id, audio_seq in enumerate(audio_seq_list):
+ data_in[sample_index] = \
+ KWS.quantize_audio(audio_seq,
+ num_bits=self.quantization['bits'],
+ compand=self.quantization['compand'],
+ mu=self.quantization['mu'])
+ data_shift_limits[sample_index] = shift_limits[local_id]
+ data_type[sample_index] = d_typ
+ sample_index += 1
dur = time.time() - time_s
print(f'Finished in {dur:.3f} seconds.')
@@ -600,19 +658,30 @@ def __gen_datasets(self, exp_len=16384, row_len=128, overlap_ratio=0):
data_in_all = data_in.copy()
data_class_all = data_class.copy()
data_type_all = data_type.copy()
+ data_shift_limits_all = data_shift_limits.copy()
else:
data_in_all = np.concatenate((data_in_all, data_in), axis=0)
data_class_all = np.concatenate((data_class_all, data_class), axis=0)
data_type_all = np.concatenate((data_type_all, data_type), axis=0)
+ data_shift_limits_all = \
+ np.concatenate((data_shift_limits_all, data_shift_limits), axis=0)
dur = time.time() - time_s
print(f'Data concatenation finished in {dur:.3f} seconds.')
data_in_all = torch.from_numpy(data_in_all)
data_class_all = torch.from_numpy(data_class_all)
data_type_all = torch.from_numpy(data_type_all)
+ data_shift_limits_all = torch.from_numpy(data_shift_limits_all)
+
+ # apply static shift & noise augmentation for validation examples
+ for sample_index in range(data_in_all.shape[0]):
+ if data_type_all[sample_index] == 2:
+ data_in_all[sample_index] = \
+ self.shift_and_noise_augment(data_in_all[sample_index],
+ data_shift_limits_all[sample_index])
- mfcc_dataset = (data_in_all, data_class_all, data_type_all)
- torch.save(mfcc_dataset, os.path.join(self.processed_folder, self.data_file))
+ raw_dataset = (data_in_all, data_class_all, data_type_all, data_shift_limits_all)
+ torch.save(raw_dataset, os.path.join(self.processed_folder, self.data_file))
print('Dataset created.')
print(f'Training: {train_count}, Validation: {valid_count}, Test: {test_count}')
@@ -658,7 +727,7 @@ def KWS_get_datasets(data, load_train=True, load_test=True, num_classes=6):
else:
raise ValueError(f'Unsupported num_classes {num_classes}')
- augmentation = {'aug_num': 2, 'shift': {'min': -0.15, 'max': 0.15},
+ augmentation = {'aug_num': 2, 'shift': {'min': -0.1, 'max': 0.1},
'noise_var': {'min': 0, 'max': 1.0}}
quantization_scheme = {'compand': False, 'mu': 10}
diff --git a/datasets/msnoise.py b/datasets/msnoise.py
index 9bfeb38fb..e2eaee42e 100644
--- a/datasets/msnoise.py
+++ b/datasets/msnoise.py
@@ -1,6 +1,7 @@
#
# Copyright (c) 2018 Intel Corporation
# Portions Copyright (C) 2019-2023 Maxim Integrated Products, Inc.
+# Portions Copyright (C) 2023-2024 Analog Devices, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -44,6 +45,7 @@ class MSnoise:
exist.
classes(array): List of keywords to be used.
d_type(string): Option for the created dataset. ``train`` or ``test``.
+ dataset_len(int): Dataset length to be returned.
remove_unknowns (bool, optional): If true, unchosen classes are not gathered as
the unknown class.
transform (callable, optional): A function/transform that takes in an PIL image
@@ -62,17 +64,23 @@ class MSnoise:
'CopyMachine': 6, 'Field': 7, 'Hallway': 8, 'Kitchen': 9,
'LivingRoom': 10, 'Metro': 11, 'Munching': 12, 'NeighborSpeaking': 13,
'Office': 14, 'Park': 15, 'Restaurant': 16, 'ShuttingDoor': 17,
- 'Square': 18, 'SqueakyChair': 19, 'Station': 20, 'Traffic': 21,
- 'Typing': 22, 'VacuumCleaner': 23, 'WasherDryer': 24, 'Washing': 25}
+ 'Square': 18, 'SqueakyChair': 19, 'Station': 20, 'TradeShow': 21, 'Traffic': 22,
+ 'Typing': 23, 'VacuumCleaner': 24, 'WasherDryer': 25, 'Washing': 26}
- def __init__(self, root, classes, d_type, remove_unknowns=False,
- transform=None, quantize=False, download=False):
+ def __init__(self, root, classes, d_type, dataset_len, exp_len=16384, fs=16000,
+ noise_time_step=0.25, remove_unknowns=False, transform=None,
+ quantize=False, download=False):
self.root = root
self.classes = classes
self.d_type = d_type
self.remove_unknowns = remove_unknowns
self.transform = transform
+ self.dataset_len = dataset_len
+ self.exp_len = exp_len
+ self.fs = fs
+ self.noise_time_step = noise_time_step
+
self.noise_train_folder = os.path.join(self.raw_folder, 'noise_train')
self.noise_test_folder = os.path.join(self.raw_folder, 'noise_test')
self.url_train = \
@@ -81,16 +89,13 @@ def __init__(self, root, classes, d_type, remove_unknowns=False,
'https://api.github.com/repos/microsoft/MS-SNSD/contents/noise_test?ref=master'
self.quantize = quantize
- if self.quantize:
- self.data_file = 'dataset_quantized.pt'
- else:
- self.data_file = 'dataset_unquantized.pt'
-
if download:
self.__download()
- self.data, self.targets, self.data_type = torch.load(os.path.join(
- self.processed_folder, self.data_file))
+ self.data, self.targets, self.data_type, self.rms_val = self.__gen_datasets()
+
+ # rms values for each sample to be returned
+ self.rms = np.zeros(self.dataset_len)
self.__filter_dtype()
self.__filter_classes()
@@ -101,24 +106,16 @@ def raw_folder(self):
"""
return os.path.join(self.root, self.__class__.__name__, 'raw')
- @property
- def processed_folder(self):
- """Folder for the processed data.
- """
- return os.path.join(self.root, self.__class__.__name__, 'processed')
-
def __download(self):
- if self.__check_exists():
+ if os.path.exists(self.raw_folder):
return
self.__makedir_exist_ok(self.noise_train_folder)
self.__makedir_exist_ok(self.noise_test_folder)
- self.__makedir_exist_ok(self.processed_folder)
self.__download_raw(self.url_train)
self.__download_raw(self.url_test)
- self.__gen_datasets()
def __download_raw(self, api_url):
opener = urllib.request.build_opener()
@@ -145,9 +142,6 @@ def __download_raw(self, api_url):
print('Interrupted while downloading!')
sys.exit()
- def __check_exists(self):
- return os.path.exists(os.path.join(self.processed_folder, self.data_file))
-
def __makedir_exist_ok(self, dirpath):
try:
os.makedirs(dirpath)
@@ -158,21 +152,25 @@ def __makedir_exist_ok(self, dirpath):
raise
def __filter_dtype(self):
+
if self.d_type == 'train':
- idx_to_select = (self.data_type == 0)[:, -1]
+ bool_list = [i == 0 for i in self.data_type]
+ idx_to_select = [i for i, x in enumerate(bool_list) if x]
elif self.d_type == 'test':
- idx_to_select = (self.data_type == 1)[:, -1]
+ bool_list = [i == 1 for i in self.data_type]
+ idx_to_select = [i for i, x in enumerate(bool_list) if x]
else:
print(f'Unknown data type: {self.d_type}')
return
- print(self.data.shape)
- self.data = self.data[idx_to_select, :]
- self.targets = self.targets[idx_to_select, :]
+ self.data = [self.data[i] for i in idx_to_select]
+ self.targets = [self.targets[i] for i in idx_to_select]
+ self.rms_val = [self.rms_val[i] for i in idx_to_select]
del self.data_type
def __filter_classes(self):
print('\n')
+ self.targets = np.array(self.targets)
initial_new_class_label = len(self.class_dict)
new_class_label = initial_new_class_label
for c in self.classes:
@@ -181,21 +179,22 @@ def __filter_classes(self):
return
# else:
print(f'Class {c}, {self.class_dict[c]}')
- num_elems = (self.targets == self.class_dict[c]).cpu().sum()
- print(f'Number of elements in class {c}: {num_elems}')
- self.targets[(self.targets == self.class_dict[c])] = new_class_label
+ bool_list = [self.class_dict[c] == i for i in self.targets]
+ idx = [i for i, x in enumerate(bool_list) if x]
+ self.targets[idx] = new_class_label
+ print(f'{c}: {new_class_label - initial_new_class_label}')
new_class_label += 1
- num_elems = (self.targets < initial_new_class_label).cpu().sum()
- print(f'Number of elements in class unknown: {num_elems}')
self.targets[(self.targets < initial_new_class_label)] = new_class_label
if self.remove_unknowns:
- idx_to_remove = (self.targets == new_class_label)[:, -1]
- idx_to_keep = torch.logical_not(idx_to_remove)
- self.data = self.data[idx_to_keep, :]
- self.targets = self.targets[idx_to_keep, :]
- self.targets -= initial_new_class_label
- print(np.unique(self.targets.data.cpu()))
+ bool_list = [i != new_class_label for i in self.targets]
+ idx_to_keep = [i for i, x in enumerate(bool_list) if x]
+
+ self.data = [self.data[i] for i in idx_to_keep]
+ self.targets = [self.targets[i] for i in idx]
+ self.rms_val = [self.rms_val[i] for i in idx]
+
+ self.targets = [target - initial_new_class_label for target in self.targets]
print('\n')
@staticmethod
@@ -210,20 +209,36 @@ def quantize_audio(data, num_bits=8):
return np.uint8(q_data)
def __len__(self):
- return len(self.data)
+ return self.dataset_len
def __getitem__(self, index):
- inp, target = self.data[index].type(torch.FloatTensor), int(self.targets[index])
+
+ rec_num = len(self.data)
+
+ rnd_num = np.random.randint(0, rec_num)
+ self.rms[index] = self.rms_val[rnd_num]
+
+ rec_len = len(self.data[rnd_num])
+
+ max_start_idx = rec_len - self.exp_len
+ start_idx = np.random.randint(0, max_start_idx)
+ end_idx = start_idx + self.exp_len
+
+ inp = self.__reshape_audio(self.data[rnd_num][start_idx:end_idx])
+ target = int(self.targets[rnd_num])
+
if self.quantize:
inp /= 256
if self.transform is not None:
inp = self.transform(inp)
return inp, target
- def __gen_datasets(self, exp_len=16384, row_len=128, overlap_ratio=0,
- noise_time_step=0.25, train_ratio=0.6):
- print('Generating dataset from raw data samples for the first time. ')
- print('Warning: This process could take 5-10 minutes!')
+ def __reshape_audio(self, audio, row_len=128):
+
+ return torch.transpose(torch.tensor(audio.reshape((-1, row_len))), 1, 0)
+
+ def __gen_datasets(self, exp_len=16384, row_len=128, overlap_ratio=0):
+
with warnings.catch_warnings():
warnings.simplefilter('error')
@@ -253,76 +268,33 @@ def __gen_datasets(self, exp_len=16384, row_len=128, overlap_ratio=0,
# Folders
train_test_folders = [self.noise_train_folder, self.noise_test_folder]
- # Determining the array sizes
- num_seqs = 0
- for label in labels:
- train_count = 0
- test_count = 0
+ data_in = []
+ data_type = []
+ data_class = []
+ rms_val = []
+
+ for i, label in enumerate(labels):
for folder in train_test_folders:
for record_name in os.listdir(folder):
if record_name.split('_')[0] in label:
record_path = os.path.join(folder, record_name)
- record, fs = librosa.load(record_path, offset=0, sr=None)
- rec_len = np.size(record)
- max_start_time = ((rec_len / fs - 1) -
- (rec_len / fs % noise_time_step))
- num_seqs += int(max_start_time / noise_time_step + 1)
- print(f'Num sequences: {num_seqs}')
-
- # Creating the empty arrays
- if self.quantize:
- data_in = np.zeros((num_seqs, row_len, num_rows), dtype=np.uint8)
- else:
- data_in = np.zeros((num_seqs, row_len, num_rows), dtype=np.float32)
- data_type = np.zeros((num_seqs, 1), dtype=np.uint8)
- data_class = np.zeros((num_seqs, 1), dtype=np.uint8)
+ record, _ = librosa.load(record_path, offset=0, sr=None)
- data_idx = 0
- for i, label in enumerate(labels):
- print(f'Processing label:{label}')
- train_count = 0
- test_count = 0
- for folder in train_test_folders:
- for record_name in sorted(os.listdir(folder)):
- if record_name.split('_')[0] in label:
- if hash(record_name) % 10 < 10*train_ratio:
- d_type = np.uint8(0) # train+val
- train_count += 1
+ if self.quantize:
+ data_in.append(self.quantize_audio(record))
else:
- d_type = np.uint8(1) # test
- test_count += 1
- record_path = os.path.join(folder, record_name)
- record, fs = librosa.load(record_path, offset=0, sr=None)
- rec_len = np.size(record)
- max_start_time = \
- (rec_len / fs - 1) - (rec_len / fs % noise_time_step)
- for start_time in np.arange(0,
- int((max_start_time+noise_time_step)*fs),
- int(noise_time_step*fs)):
- end_time = start_time + fs
- audio_seq = record[start_time:end_time]
- data_type[data_idx, 0] = d_type
- data_class[data_idx, 0] = i
- for n_r in range(num_rows):
- start_idx = n_r*(row_len - overlap)
- end_idx = start_idx + row_len
- audio_chunk = audio_seq[start_idx:end_idx]
- audio_chunk = \
- np.pad(audio_chunk, [0, row_len-audio_chunk.size])
- if self.quantize:
- data_in[data_idx, :, n_r] = \
- self.quantize_audio(audio_chunk)
- else:
- data_in[data_idx, :, n_r] = audio_chunk
- data_idx += 1
-
- data_in = torch.from_numpy(data_in)
- data_class = torch.from_numpy(data_class)
- data_type = torch.from_numpy(data_type)
-
- noise_dataset = (data_in, data_class, data_type)
- torch.save(noise_dataset, os.path.join(self.processed_folder, self.data_file))
- print('Dataset created!')
+ data_in.append(record)
+
+ if folder == self.noise_train_folder:
+ data_type.append(0) # train + val
+ elif folder == self.noise_test_folder:
+ data_type.append(1) # test
+
+ data_class.append(i)
+ rms_val.append(np.mean(record**2)**0.5)
+
+ noise_dataset = (data_in, data_class, data_type, rms_val)
+ return noise_dataset
def MSnoise_get_datasets(data, load_train=True, load_test=True):
@@ -338,9 +310,11 @@ def MSnoise_get_datasets(data, load_train=True, load_test=True):
classes = ['AirConditioner', 'AirportAnnouncements',
'Babble', 'Bus', 'CafeTeria', 'Car',
- 'CopyMachine', 'Metro',
- 'Office', 'Restaurant', 'ShuttingDoor',
- 'Traffic', 'Typing', 'VacuumCleaner', 'Washing']
+ 'CopyMachine', 'Field', 'Hallway', 'Kitchen',
+ 'LivingRoom', 'Metro', 'Munching', 'NeighborSpeaking',
+ 'Office', 'Park', 'Restaurant', 'ShuttingDoor',
+ 'Square', 'SqueakyChair', 'Station', 'Traffic',
+ 'Typing', 'VacuumCleaner', 'WasherDryer', 'Washing', 'TradeShow']
remove_unknowns = True
transform = transforms.Compose([
@@ -349,14 +323,14 @@ def MSnoise_get_datasets(data, load_train=True, load_test=True):
quantize = True
if load_train:
- train_dataset = MSnoise(root=data_dir, classes=classes, d_type='train',
+ train_dataset = MSnoise(root=data_dir, classes=classes, d_type='train', dataset_len=11005,
remove_unknowns=remove_unknowns, transform=transform,
quantize=quantize, download=True)
else:
train_dataset = None
if load_test:
- test_dataset = MSnoise(root=data_dir, classes=classes, d_type='test',
+ test_dataset = MSnoise(root=data_dir, classes=classes, d_type='test', dataset_len=11005,
remove_unknowns=remove_unknowns, transform=transform,
quantize=quantize, download=True)
@@ -378,33 +352,27 @@ def MSnoise_get_unquantized_datasets(data, load_train=True, load_test=True):
"""
(data_dir, args) = data
- # classes = ['AirConditioner', 'AirportAnnouncements',
- # 'Babble', 'Bus', 'CafeTeria', 'Car',
- # 'CopyMachine', 'Field', 'Hallway', 'Kitchen',
- # 'LivingRoom', 'Metro', 'Munching', 'NeighborSpeaking',
- # 'Office', 'Park', 'Restaurant', 'ShuttingDoor',
- # 'Square', 'SqueakyChair', 'Station', 'Traffic',
- # 'Typing', 'VacuumCleaner', 'WasherDryer', 'Washing']
-
- classes = ['AirConditioner',
- 'CafeTeria', 'Car',
- 'CopyMachine',
- 'Office', 'Restaurant',
- 'Typing', 'VacuumCleaner', 'WasherDryer']
+ classes = ['AirConditioner', 'AirportAnnouncements',
+ 'Babble', 'Bus', 'CafeTeria', 'Car',
+ 'CopyMachine', 'Field', 'Hallway', 'Kitchen',
+ 'LivingRoom', 'Metro', 'Munching', 'NeighborSpeaking',
+ 'Office', 'Park', 'Restaurant', 'ShuttingDoor',
+ 'Square', 'SqueakyChair', 'Station', 'Traffic',
+ 'Typing', 'VacuumCleaner', 'WasherDryer', 'Washing', 'TradeShow']
remove_unknowns = True
transform = None
quantize = False
if load_train:
- train_dataset = MSnoise(root=data_dir, classes=classes, d_type='train',
+ train_dataset = MSnoise(root=data_dir, classes=classes, d_type='train', dataset_len=11005,
remove_unknowns=remove_unknowns, transform=transform,
quantize=quantize, download=True)
else:
train_dataset = None
if load_test:
- test_dataset = MSnoise(root=data_dir, classes=classes, d_type='test',
+ test_dataset = MSnoise(root=data_dir, classes=classes, d_type='test', dataset_len=11005,
remove_unknowns=remove_unknowns, transform=transform,
quantize=quantize, download=True)
diff --git a/datasets/signalmixer.py b/datasets/signalmixer.py
new file mode 100644
index 000000000..68845f1ae
--- /dev/null
+++ b/datasets/signalmixer.py
@@ -0,0 +1,136 @@
+#
+# Copyright (c) 2018 Intel Corporation
+# Portions Copyright (C) 2019-2023 Maxim Integrated Products, Inc.
+# Portions Copyright (C) 2023-2024 Analog Devices, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+Classes and functions used to create noisy keyword spotting dataset.
+"""
+import numpy as np
+import torch
+
+
+class signalmixer:
+ """
+ Signal mixer dataloader to create datasets with specified
+ length using a noise dataset and a speech dataset and a specified SNR level.
+
+ Args:
+ signal_dataset(object): KWS dataset object.
+ snr(int): SNR level to be created in the mixed dataset.
+ noise_kind(string): Noise kind that will be applied to the speech dataset.
+ noise_dataset(object, optional): MSnoise dataset object.
+ """
+
+ def __init__(self, signal_dataset, snr, noise_kind, noise_dataset=None):
+
+ self.signal_data = signal_dataset.data
+ self.signal_targets = signal_dataset.targets
+
+ if noise_kind != 'WhiteNoise':
+ self.noise_data = noise_dataset.data
+ self.noise_targets = noise_dataset.targets
+
+ # using getitem to reach the noise test data
+ self.noise_dataset_float = next(iter(torch.utils.data.DataLoader(
+ noise_dataset, batch_size=noise_dataset.dataset_len)))[0]
+
+ self.noise_rms = noise_dataset.rms
+
+ self.snr = snr
+ self.noise_kind = noise_kind
+
+ # using getitem to reach the speech test data
+ self.test_dataset_float = next(iter(torch.utils.data.DataLoader(
+ signal_dataset, batch_size=signal_dataset.data.shape[0])))[0]
+
+ if noise_kind == 'WhiteNoise':
+ self.mixed_signal = self.white_noise_mixer()
+ else:
+ self.mixed_signal = self.snr_mixer()
+
+ def __getitem__(self, index):
+
+ inp = self.mixed_signal[index].type(torch.FloatTensor)
+ target = int(self.signal_targets[index])
+ return inp, target
+
+ def __len__(self):
+ return len(self.mixed_signal)
+
+ def snr_mixer(self):
+ ''' creates mixed signal dataset using the SNR level and the noise dataset
+ '''
+ clean = self.test_dataset_float
+ noise = self.noise_dataset_float
+
+ idx = np.random.randint(0, noise.shape[0], clean.shape[0])
+ noise = noise[idx]
+ rms_noise = self.noise_rms[idx]
+
+ snr = self.snr
+
+ rmsclean = torch.sqrt(torch.mean(clean.reshape(
+ clean.shape[0], -1)**2, 1, keepdims=True)).unsqueeze(1)
+ scalarclean = 1 / rmsclean
+ clean = clean * scalarclean
+
+ scalarnoise = 1 / rms_noise.reshape(-1, 1, 1)
+ noise = noise * scalarnoise
+
+ cleanfactor = 10**(snr/20)
+ noisyspeech = cleanfactor * clean + noise
+ noisyspeech = noisyspeech / (torch.tensor(scalarnoise) + cleanfactor * scalarclean)
+
+ # 16384 --> (noisyspeech[0].shape[0])*(noisyspeech[0].shape[1])
+ speech_shape = noisyspeech[0].shape[0]*noisyspeech[0].shape[1]
+ max_mixed = torch.max(abs(noisyspeech.reshape(
+ noisyspeech.shape[0], speech_shape)), 1, keepdims=True).values
+
+ noisyspeech = noisyspeech * (1 / max_mixed).unsqueeze(1)
+ return noisyspeech
+
+ def white_noise_mixer(self):
+
+ '''creates mixed signal dataset using the SNR level and white noise
+ '''
+ clean = self.test_dataset_float
+ snr = self.snr
+
+ mean = 0
+ std = 1
+ noise = np.random.normal(mean, std, clean.shape)
+ noise = torch.tensor(noise, dtype=torch.float32)
+
+ rmsclean = (torch.mean(clean.reshape(
+ clean.shape[0], -1)**2, 1, keepdims=True)**0.5).unsqueeze(1)
+ scalarclean = 1 / rmsclean
+ clean = clean * scalarclean
+
+ rmsnoise = (torch.mean(noise.reshape(
+ noise.shape[0], -1)**2, 1, keepdims=True)**0.5).unsqueeze(1)
+ scalarnoise = 1 / rmsnoise
+ noise = noise * scalarnoise
+
+ cleanfactor = 10**(snr/20)
+ noisyspeech = cleanfactor * clean + noise
+ noisyspeech = noisyspeech / (scalarnoise + cleanfactor * scalarclean)
+
+ # scaling to ~[-1,1]
+ max_mixed = torch.max(abs(noisyspeech.reshape(
+ noisyspeech.shape[0], 16384)), 1, keepdims=True).values
+ noisyspeech = noisyspeech * (1 / max_mixed).unsqueeze(1)
+
+ return noisyspeech
diff --git a/models/ai85net-kws20-nas.py b/models/ai85net-kws20-nas.py
new file mode 100644
index 000000000..fa6d108aa
--- /dev/null
+++ b/models/ai85net-kws20-nas.py
@@ -0,0 +1,107 @@
+###################################################################################################
+#
+# Copyright (C) 2023-2024 Analog Devices, Inc. All Rights Reserved.
+#
+# Analog Devices, Inc. Default Copyright Notice:
+# https://www.analog.com/en/about-adi/legal-and-risk-oversight/intellectual-property/copyright-notice.html
+#
+###################################################################################################
+#
+# Copyright (C) 2021-2023 Maxim Integrated Products, Inc. All Rights Reserved.
+#
+# Maxim Integrated Products, Inc. Default Copyright Notice:
+# https://www.maximintegrated.com/en/aboutus/legal/copyrights.html
+#
+###################################################################################################
+"""
+Keyword spotting network for AI85
+"""
+from torch import nn
+
+import ai8x
+
+
+class AI85KWS20NetNAS(nn.Module):
+ """
+ KWS20 NAS Audio net, found via Neural Architecture Search
+ It significantly outperforms earlier networks (v1, v2, v3), though with a higher
+ parameter count and slightly increased latency.
+ """
+
+ # num_classes = n keywords + 1 unknown
+ def __init__(
+ self,
+ num_classes=21,
+ num_channels=128,
+ dimensions=(128, 1), # pylint: disable=unused-argument
+ bias=True,
+ **kwargs
+ ):
+ super().__init__()
+ self.conv1_1 = ai8x.FusedConv1dBNReLU(num_channels, 128, 1, stride=1, padding=0,
+ bias=bias, batchnorm="NoAffine", **kwargs)
+ self.conv1_2 = ai8x.FusedConv1dBNReLU(128, 64, 3, stride=1, padding=1,
+ bias=bias, batchnorm="NoAffine", **kwargs)
+ self.conv1_3 = ai8x.FusedConv1dBNReLU(64, 128, 3, stride=1, padding=1,
+ bias=bias, batchnorm="NoAffine", **kwargs)
+ self.conv2_1 = ai8x.FusedMaxPoolConv1dBNReLU(128, 128, 3, stride=1, padding=1,
+ bias=bias, batchnorm="NoAffine", **kwargs)
+ self.conv2_2 = ai8x.FusedConv1dBNReLU(128, 64, 1, stride=1, padding=0,
+ bias=bias, batchnorm="NoAffine", **kwargs)
+ self.conv2_3 = ai8x.FusedConv1dBNReLU(64, 128, 1, stride=1, padding=0,
+ bias=bias, batchnorm="NoAffine", **kwargs)
+ self.conv3_1 = ai8x.FusedMaxPoolConv1dBNReLU(128, 128, 3, stride=1, padding=1,
+ bias=bias, batchnorm="NoAffine", **kwargs)
+ self.conv3_2 = ai8x.FusedConv1dBNReLU(128, 64, 5, stride=1, padding=2,
+ bias=bias, batchnorm="NoAffine", **kwargs)
+ self.conv4_1 = ai8x.FusedMaxPoolConv1dBNReLU(64, 128, 5, stride=1, padding=2,
+ bias=bias, batchnorm="NoAffine", **kwargs)
+ self.conv4_2 = ai8x.FusedConv1dBNReLU(128, 128, 1, stride=1, padding=0,
+ bias=bias, batchnorm="NoAffine", **kwargs)
+ self.conv5_1 = ai8x.FusedMaxPoolConv1dBNReLU(128, 128, 5, stride=1, padding=2,
+ bias=bias, batchnorm="NoAffine", **kwargs)
+ self.conv5_2 = ai8x.FusedConv1dBNReLU(128, 64, 3, stride=1, padding=1,
+ bias=bias, batchnorm="NoAffine", **kwargs)
+ self.conv6_1 = ai8x.FusedMaxPoolConv1dBNReLU(64, 64, 5, stride=1, padding=2,
+ bias=bias, batchnorm="NoAffine", **kwargs)
+ self.conv6_2 = ai8x.FusedConv1dBNReLU(64, 128, 1, stride=1, padding=0,
+ bias=bias, batchnorm="NoAffine", **kwargs)
+ self.fc = ai8x.Linear(512, num_classes, bias=bias, wide=True, **kwargs)
+
+ def forward(self, x): # pylint: disable=arguments-differ
+ """Forward prop"""
+ # Run CNN
+ x = self.conv1_1(x)
+ x = self.conv1_2(x)
+ x = self.conv1_3(x)
+ x = self.conv2_1(x)
+ x = self.conv2_2(x)
+ x = self.conv2_3(x)
+ x = self.conv3_1(x)
+ x = self.conv3_2(x)
+ x = self.conv4_1(x)
+ x = self.conv4_2(x)
+ x = self.conv5_1(x)
+ x = self.conv5_2(x)
+ x = self.conv6_1(x)
+ x = self.conv6_2(x)
+ x = x.view(x.size(0), -1)
+ x = self.fc(x)
+ return x
+
+
+def ai85kws20netnas(pretrained=False, **kwargs):
+ """
+ Constructs a AI85KWS20NetNAS model.
+ """
+ assert not pretrained
+ return AI85KWS20NetNAS(**kwargs)
+
+
+models = [
+ {
+ 'name': 'ai85kws20netnas',
+ 'min_input': 1,
+ 'dim': 1,
+ },
+]
diff --git a/notebooks/Bayer2RGB_Evaluation.ipynb b/notebooks/Bayer2RGB_Evaluation.ipynb
index 24059aec0..c793e444f 100644
--- a/notebooks/Bayer2RGB_Evaluation.ipynb
+++ b/notebooks/Bayer2RGB_Evaluation.ipynb
@@ -22,8 +22,10 @@
"source": [
"###################################################################################################\n",
"#\n",
- "# Copyright © 2023 Analog Devices, Inc. All Rights Reserved.\n",
- "# This software is proprietary and confidential to Analog Devices, Inc. and its licensors.\n",
+ "# Copyright (C) 2023-2024 Analog Devices, Inc. All Rights Reserved.\n",
+ "#\n",
+ "# Analog Devices, Inc. Default Copyright Notice:\n",
+ "# https://www.analog.com/en/about-adi/legal-and-risk-oversight/intellectual-property/copyright-notice.html\n",
"#\n",
"###################################################################################################import cv2\n",
"import importlib\n",
diff --git a/notebooks/KWS_Noise_Evaluation.ipynb b/notebooks/KWS_Noise_Evaluation.ipynb
new file mode 100644
index 000000000..934c52387
--- /dev/null
+++ b/notebooks/KWS_Noise_Evaluation.ipynb
@@ -0,0 +1,868 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Automated Evaluation for KWS Under Different Noise Scenarios\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Using this notebook, you can test previously trained KWS models under various noise types at desired SNR levels. You'll need:\n",
+ "- the trained model's checkpoint file\n",
+ "- a list specifying the types of noise\n",
+ "- a list specifying the SNR levels.\n",
+ "\n",
+ "This notebook uses the signalmixer data loader to mix the KWS and specified noise data, and creates a mixed dataset using the specified SNR level. The notebook performs the evaluation on these mixed datasets and creates comparison plots for different types of models."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "###################################################################################################\n",
+ "#\n",
+ "# Copyright (C) 2023-2024 Analog Devices, Inc. All Rights Reserved.\n",
+ "#\n",
+ "# Analog Devices, Inc. Default Copyright Notice:\n",
+ "# https://www.analog.com/en/about-adi/legal-and-risk-oversight/intellectual-property/copyright-notice.html\n",
+ "#\n",
+ "###################################################################################################\n",
+ "#\n",
+ "# Copyright (C) 2022-2023 Maxim Integrated Products, Inc. All Rights Reserved.\n",
+ "#\n",
+ "# Maxim Integrated Products, Inc. Default Copyright Notice:\n",
+ "# https://www.maximintegrated.com/en/aboutus/legal/copyrights.html\n",
+ "#\n",
+ "###################################################################################################\n",
+ "\n",
+ "import os\n",
+ "import sys\n",
+ "\n",
+ "import numpy as np\n",
+ "import torch\n",
+ "import pandas as pd\n",
+ "import matplotlib.pyplot as plt\n",
+ "\n",
+ "import torchnet.meter as tnt\n",
+ "from collections import OrderedDict\n",
+ "import importlib\n",
+ "from torchvision import transforms\n",
+ "\n",
+ "sys.path.append(os.path.join(os.getcwd(), '..'))\n",
+ "sys.path.append(os.path.join(os.getcwd(), '..', 'models'))\n",
+ "sys.path.append(os.path.join(os.getcwd(), '..', 'datasets'))\n",
+ "\n",
+ "from types import SimpleNamespace\n",
+ "\n",
+ "import ai8x\n",
+ "\n",
+ "import msnoise\n",
+ "from signalmixer import signalmixer"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Step 1: Initialize and load the model"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "First, set the checkpoint path and the corresponding model name. "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "trained_checkpoint_path = os.path.join('..', '..', 'ai8x-synthesis', 'trained', 'ai85-kws20_nas-qat8.pth.tar')\n",
+ "mod = importlib.import_module(\"ai85net-kws20-nas\")\n",
+ "model_file = \"ai85net-kws20-nas\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Working with device: cuda\n",
+ "Configuring device: MAX78000, simulate=False.\n"
+ ]
+ }
+ ],
+ "source": [
+ "dataset = importlib.import_module(\"kws20\")\n",
+ "\n",
+ "classes = ['up', 'down', 'left', 'right', 'stop', 'go', 'yes', 'no', 'on', 'off', 'one',\n",
+ " 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'zero', 'unknown']\n",
+ "\n",
+ "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
+ "print(\"Working with device:\", device)\n",
+ "\n",
+ "ai8x.set_device(device=85, simulate=False, round_avg=False)\n",
+ "qat_policy = {'start_epoch': 10, 'weight_bits': 8, 'bias_bits': 8}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "model = mod.AI85KWS20NetNAS(num_classes=len(classes), num_channels=128, dimensions=(128, 1), bias=True, \n",
+ " quantize_activation=False)\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "checkpoint = torch.load(trained_checkpoint_path)\n",
+ "\n",
+ "state_dict = checkpoint['state_dict']\n",
+ "new_state_dict = OrderedDict()\n",
+ "for k, v in state_dict.items():\n",
+ " if k.startswith('module.'):\n",
+ " k = k[7:]\n",
+ " new_state_dict[k] = v\n",
+ "checkpoint['state_dict'] = new_state_dict\n",
+ "\n",
+ "ai8x.fuse_bn_layers(model)\n",
+ "ai8x.initiate_qat(model, qat_policy)\n",
+ "model.load_state_dict(checkpoint['state_dict'], strict=False)\n",
+ "\n",
+ "ai8x.update_model(model)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Step 2: Set up the test set and evaluation parameters"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Then, load the KWS test set, specify the noise list and the SNR level list. \n",
+ "\n",
+ "Note: Noise types should be picked from the available classes within the MSnoise dataset (`datasets/msnoise.py`)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\n",
+ "Processing test...\n",
+ "test set: 11005 elements\n",
+ "Class up (# 31): 425 elements\n",
+ "Class down (# 5): 406 elements\n",
+ "Class left (# 15): 412 elements\n",
+ "Class right (# 23): 396 elements\n",
+ "Class stop (# 27): 411 elements\n",
+ "Class go (# 11): 402 elements\n",
+ "Class yes (# 34): 419 elements\n",
+ "Class no (# 19): 405 elements\n",
+ "Class on (# 21): 396 elements\n",
+ "Class off (# 20): 402 elements\n",
+ "Class one (# 22): 399 elements\n",
+ "Class two (# 30): 424 elements\n",
+ "Class three (# 28): 405 elements\n",
+ "Class four (# 10): 400 elements\n",
+ "Class five (# 7): 445 elements\n",
+ "Class six (# 26): 394 elements\n",
+ "Class seven (# 24): 406 elements\n",
+ "Class eight (# 6): 408 elements\n",
+ "Class nine (# 18): 408 elements\n",
+ "Class zero (# 35): 418 elements\n",
+ "Class UNKNOWN: 2824 elements\n"
+ ]
+ }
+ ],
+ "source": [
+ "sn = SimpleNamespace()\n",
+ "sn.truncate_testset = False\n",
+ "sn.act_mode_8bit = False\n",
+ "\n",
+ "data_path = '/data'\n",
+ "\n",
+ "_, test_dataset = dataset.KWS_20_get_datasets( (data_path, sn), load_train=False, load_test=True)\n",
+ "\n",
+ "originals = list(range(0, len(test_dataset), 3))\n",
+ "test_dataset.data = test_dataset.data[originals]\n",
+ "test_dataset.targets = test_dataset.targets[originals]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "noise_list = ['AirConditioner',\n",
+ " 'AirportAnnouncements',\n",
+ " 'Babble',\n",
+ " 'CopyMachine',\n",
+ " 'Munching',\n",
+ " 'NeighborSpeaking',\n",
+ " 'ShuttingDoor',\n",
+ " 'Typing',\n",
+ " 'VacuumCleaner',\n",
+ " 'TradeShow',\n",
+ " 'WhiteNoise']"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "snr_list = [-5, 0, 5, 10, 15, 20]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Step 3: Define functions to evaluate the model under different SNR levels"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Using the `evaluate`, `snr_testing` and `benchmark` functions, you can evaluate the trained model under different SNR levels.\n",
+ "\n",
+ "- The `evaluate` function evaluates the model under the specified SNR level and noise type.\n",
+ "- The `snr_testing` function executes a loop over the SNR list. It calls the `evaluate` function for each SNR level, for the specified noise type.\n",
+ "- The `benchmark` function executes a loop over the list of noise types. It calls the `snr_testing` function for each type of noise."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def evaluate(db = None, noise = False, noise_type = None):\n",
+ "\n",
+ " if (noise == True) and (noise_type == None):\n",
+ " print('Noise type is not specified. Noise will not be applied.')\n",
+ " noise = False\n",
+ "\n",
+ " model.eval()\n",
+ " model.to(device) \n",
+ " classerr = tnt.ClassErrorMeter(accuracy=True, topk=(1, min(len(classes), 5)))\n",
+ "\n",
+ " transform = transforms.Compose([\n",
+ " ai8x.normalize(args=sn)\n",
+ " ])\n",
+ " \n",
+ " if noise:\n",
+ " if (noise_type == 'WhiteNoise'):\n",
+ "\n",
+ " mixed_signals = signalmixer(test_dataset, snr = db, noise_type = noise_type, noise_dataset = None)\n",
+ " mixed_signals_loader = torch.utils.data.DataLoader(mixed_signals, batch_size = 256)\n",
+ "\n",
+ " else:\n",
+ " noise_dataset = msnoise.MSnoise(root = data_path, classes = [noise_type], d_type = 'test', dataset_len = 11005, remove_unknowns=True,\n",
+ " transform=None, quantize=False, download=False)\n",
+ "\n",
+ " mixed_signals = signalmixer(test_dataset, snr = db, noise_type = noise_type, noise_dataset = noise_dataset)\n",
+ " mixed_signals_loader = torch.utils.data.DataLoader(mixed_signals, batch_size = 256)\n",
+ " else:\n",
+ " mixed_signals_loader = torch.utils.data.DataLoader(test_dataset, batch_size = 256)\n",
+ " \n",
+ " with torch.no_grad():\n",
+ " for batch_idx, (inputs, targets) in enumerate(mixed_signals_loader):\n",
+ " inputs = inputs.to(device)\n",
+ " targets = targets.to(device)\n",
+ " outputs = model(inputs)\n",
+ " classerr.add(outputs, targets)\n",
+ "\n",
+ " print(\"Batch: [\",batch_idx*256 ,\"/\", len(test_dataset),\"]\")\n",
+ " acc = classerr.value()[0]\n",
+ " print(\"Accuracy: \", acc)\n",
+ " \n",
+ " print(\"Total Accuracy: \", acc)\n",
+ " return acc"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def snr_testing(snr_list = None, noise_type = None, noise = False):\n",
+ "\n",
+ " # raw test set evaluation\n",
+ " if noise == False:\n",
+ " db = None\n",
+ " noise_type = None\n",
+ " accuracies = np.zeros(1)\n",
+ " accuracies[0] = evaluate(db, noise, noise_type)\n",
+ "\n",
+ " # noisy test set evaluation\n",
+ " else:\n",
+ " accuracies = np.zeros(len(snr_list))\n",
+ "\n",
+ " for idx, db in enumerate(snr_list):\n",
+ " print(\"Evaluating SNR levels of\", db)\n",
+ " accuracies[idx] = evaluate(db, noise, noise_type)\n",
+ "\n",
+ " return accuracies"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def benchmark(noise_list = None, snr_list = None):\n",
+ "\n",
+ " if noise_list == None:\n",
+ " print('Noise type is not specified. Noise will not be applied.')\n",
+ " noise = False\n",
+ " snr_list = None\n",
+ " else:\n",
+ " noise = True\n",
+ " if snr_list == None:\n",
+ " print('Using default values for SNR levels: [-5, 20] dB.')\n",
+ " snr_list = range(-5, 20)\n",
+ "\n",
+ " if noise:\n",
+ "\n",
+ " accuracies = np.zeros((len(noise_list) + 1, len(snr_list)))\n",
+ "\n",
+ " for idx, n in enumerate(noise_list):\n",
+ " print(f'{n} Noise Evaluation')\n",
+ " accuracies[idx] = snr_testing(snr_list, noise_type = n, noise = noise)\n",
+ "\n",
+ " accuracies[-1] = snr_testing(noise = False)\n",
+ "\n",
+ " return accuracies "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "accuracies_nas = benchmark(noise_list = noise_list, snr_list = snr_list)\n",
+ "accuracies = [accuracies_nas]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Optional Step: Adding different models for comparison"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "- v3 Model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "trained_checkpoint_path = os.path.join('..', '..', 'ai8x-synthesis', 'trained', 'ai85-kws20_v3-qat8.pth.tar') \n",
+ "mod = importlib.import_module(\"ai85net-kws20-v3\")\n",
+ "model_file = \"ai85net-kws20-v3\"\n",
+ "\n",
+ "checkpoint = torch.load(trained_checkpoint_path)\n",
+ "\n",
+ "model = mod.AI85KWS20Netv3(num_classes=len(classes), num_channels=128, dimensions=(128, 1), bias=False, \n",
+ " quantize_activation=False)\n",
+ "\n",
+ "state_dict = checkpoint['state_dict']\n",
+ "new_state_dict = OrderedDict()\n",
+ "for k, v in state_dict.items():\n",
+ " if k.startswith('module.'):\n",
+ " k = k[7:]\n",
+ " new_state_dict[k] = v\n",
+ "checkpoint['state_dict'] = new_state_dict\n",
+ "\n",
+ "ai8x.fuse_bn_layers(model)\n",
+ "ai8x.initiate_qat(model, qat_policy)\n",
+ "model.load_state_dict(checkpoint['state_dict'], strict=False)\n",
+ "\n",
+ "ai8x.update_model(model)\n",
+ "\n",
+ "accuracies_v3 = benchmark(noise_list = noise_list, snr_list = snr_list)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "- v2 Model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "trained_checkpoint_path = os.path.join('..', '..', 'ai8x-synthesis', 'trained', 'ai85-kws20_v2-qat8.pth.tar')\n",
+ "mod = importlib.import_module(\"ai85net-kws20-v2\")\n",
+ "model_file = \"ai85net-kws20-v2\"\n",
+ "\n",
+ "checkpoint = torch.load(trained_checkpoint_path)\n",
+ "\n",
+ "model = mod.AI85KWS20Netv2(num_classes=len(classes), num_channels=128, dimensions=(128, 1), bias=False, \n",
+ " quantize_activation=False)\n",
+ "\n",
+ "state_dict = checkpoint['state_dict']\n",
+ "new_state_dict = OrderedDict()\n",
+ "for k, v in state_dict.items():\n",
+ " if k.startswith('module.'):\n",
+ " k = k[7:]\n",
+ " new_state_dict[k] = v\n",
+ "checkpoint['state_dict'] = new_state_dict\n",
+ "\n",
+ "ai8x.fuse_bn_layers(model)\n",
+ "ai8x.initiate_qat(model, qat_policy)\n",
+ "model.load_state_dict(checkpoint['state_dict'], strict=False)\n",
+ "\n",
+ "ai8x.update_model(model)\n",
+ "\n",
+ "accuracies_v2 = benchmark(noise_list = noise_list, snr_list = snr_list)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "accuracies = [accuracies_nas, accuracies_v2, accuracies_v3]\n",
+ "model_files = [\"ai85net-kws20-nas\", \"ai85net-kws20-v2\", \"ai85net-kws20-v3\"]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Step 4: Comparing noise types (for specified models)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "You can create data frames to examine the results of evaluation. Using the data frame, you can create plots to compare noise types for each model."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "accs = []\n",
+ "for model_name, model_acc in enumerate(accuracies):\n",
+ " csv = {}\n",
+ "\n",
+ " for idx, i in enumerate(noise_list):\n",
+ " csv[i] = model_acc[idx]\n",
+ " \n",
+ " csv_list = []\n",
+ " csv_list.append(['raw', 'None', model_acc[-1][0]])\n",
+ "\n",
+ " for i in csv.keys():\n",
+ " for idx, j in enumerate(csv[i]):\n",
+ " csv_list.append([i, snr_list[idx], j])\n",
+ "\n",
+ " df = pd.DataFrame(csv_list, columns = ['Type', 'SNR (dB)', f'{model_files[model_name]}'])\n",
+ " \n",
+ " accs.append(df)\n",
+ "\n",
+ "for i in accs:\n",
+ " df[i.columns[-1]] = i[i.columns[-1]]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " Type | \n",
+ " SNR (dB) | \n",
+ " ai85net-kws20-v3 | \n",
+ " ai85net-kws20-nas | \n",
+ " ai85net-kws20-v2 | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " raw | \n",
+ " None | \n",
+ " 89.860998 | \n",
+ " 94.140093 | \n",
+ " 91.550831 | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " AirConditioner | \n",
+ " -5 | \n",
+ " 34.723358 | \n",
+ " 39.765604 | \n",
+ " 33.878441 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " AirConditioner | \n",
+ " 0 | \n",
+ " 56.146089 | \n",
+ " 66.775688 | \n",
+ " 59.798310 | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " AirConditioner | \n",
+ " 5 | \n",
+ " 71.954211 | \n",
+ " 81.793404 | \n",
+ " 77.541564 | \n",
+ "
\n",
+ " \n",
+ " | 4 | \n",
+ " AirConditioner | \n",
+ " 10 | \n",
+ " 81.330063 | \n",
+ " 89.561188 | \n",
+ " 85.799945 | \n",
+ "
\n",
+ " \n",
+ " | ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ " ... | \n",
+ "
\n",
+ " \n",
+ " | 62 | \n",
+ " WhiteNoise | \n",
+ " 0 | \n",
+ " 71.490869 | \n",
+ " 84.655219 | \n",
+ " 80.703189 | \n",
+ "
\n",
+ " \n",
+ " | 63 | \n",
+ " WhiteNoise | \n",
+ " 5 | \n",
+ " 82.120469 | \n",
+ " 90.024530 | \n",
+ " 87.135459 | \n",
+ "
\n",
+ " \n",
+ " | 64 | \n",
+ " WhiteNoise | \n",
+ " 10 | \n",
+ " 86.944672 | \n",
+ " 92.504770 | \n",
+ " 89.533933 | \n",
+ "
\n",
+ " \n",
+ " | 65 | \n",
+ " WhiteNoise | \n",
+ " 15 | \n",
+ " 88.770782 | \n",
+ " 93.622240 | \n",
+ " 91.142001 | \n",
+ "
\n",
+ " \n",
+ " | 66 | \n",
+ " WhiteNoise | \n",
+ " 20 | \n",
+ " 88.689016 | \n",
+ " 93.976560 | \n",
+ " 91.196511 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
67 rows × 5 columns
\n",
+ "