-
Notifications
You must be signed in to change notification settings - Fork 0
/
common.py
139 lines (103 loc) · 4.3 KB
/
common.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
import math
import os
import pickle
from glob import glob
import tensorflow as tf
from constants import TFRECORDS_FORMAT_PATTERN, TFRECORDS_SAVE_PATH
def mel_spectrogram_labeled_parser(serialized_example):
features = tf.io.parse_single_example(
serialized_example,
features={
"samples": tf.io.FixedLenFeature([128 * 44], tf.float32),
"label": tf.io.FixedLenFeature([], tf.int64),
})
features["samples"] = tf.reshape(features["samples"], [128, 44]) # * (2. / 255) - 1
return features["samples"], features["label"]
def mel_spectrogram_unlabeled_parser(serialized_example):
return mel_spectrogram_labeled_parser(serialized_example)[0]
def make_input_local(filenames, parser_fn, shuffle=False, repeat=False):
def input_fn(params):
batch_size = params["batch_size"]
dataset = tf.data.TFRecordDataset(filenames)
dataset = dataset.map(parser_fn, num_parallel_calls=tf.data.experimental.AUTOTUNE)
dataset = dataset.cache()
if repeat:
dataset = dataset.repeat()
if shuffle:
dataset = dataset.shuffle(buffer_size=70000, reshuffle_each_iteration=True)
if batch_size:
dataset = dataset.batch(batch_size)
if repeat:
dataset = dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)
return dataset
return input_fn
def get_input_fn_and_steps_per_epoch(set_name, parser_fn, tfrecords_path, batch_size, sets_count):
assert set_name in ('train', 'validation', 'test', 'prediction')
steps_per_epoch = None
if batch_size:
steps_per_epoch = math.ceil(sets_count[set_name] / batch_size)
generator = make_input_local(
glob(os.path.join(tfrecords_path, TFRECORDS_FORMAT_PATTERN.format(set_name, '*', '*'))),
parser_fn,
shuffle=True if set_name == 'train' else False,
repeat=True if set_name == 'train' else False)({'batch_size': batch_size})
return generator, steps_per_epoch
# def main():
# sample_rate = 16000
# with tf.Graph().as_default() as graph:
# input_fn = make_input_local(glob(os.path.join(TFRECORDS_SAVE_PATH, TFRECORDS_FORMAT_PATTERN.format('*', '*', '*'))), shuffle=True, repeat=True)({"batch_size": 8})
#
# with tf.Session() as sess:
# init = tf.global_variables_initializer()
# sess.run(init)
#
# flag = True
# i = 1
# while flag:
# try:
# output = sess.run([input_fn])
#
# for i in range(len(output[0][1])):
# id_to_labels, _ = get_labels_dicts(KAGGLE_LABELS)
# label = id_to_labels[output[0][1][i].item()]
#
# if label in ("unknown", "silence"):
# continue
#
# sd.play(output[0][0][0], sample_rate)
# time.sleep(1)
# sd.stop()
#
# pass
# # flag = False
# # break
# # plt.figure(figsize=(12, 4))
# # librosa.display.specshow(output[0][0][i], sr=sample_rate, x_axis='time', y_axis='mel')
# # plt.title('Mel power spectrogram of ' + label)
# # plt.colorbar(format='%+02.0f dB')
# # plt.tight_layout()
# # i = i + 1
# # if i == 3:
# # flag = False
# # break
#
# except tf.errors.OutOfRangeError:
# break
if __name__ == '__main__':
# main()
pass
def load_pickle_file(filename):
with tf.io.gfile.GFile(os.path.join(TFRECORDS_SAVE_PATH, '../', filename), mode='rb') as f:
return pickle.load(f)
def load_labels():
id_to_labels = load_pickle_file('id_to_labels.pickle')
labels_to_id = load_pickle_file('labels_to_id.pickle')
return id_to_labels, labels_to_id
def load_sets_count():
sets_count = load_pickle_file('sets_count.pickle')
return sets_count
def is_valid_file(parser, arg):
if not os.path.exists(arg):
parser.error("The file %s does not exist!" % arg)
else:
return open(arg, 'r') # return an open file handle