From 21dba2cda916576355d92eb110b721b1b72e84a5 Mon Sep 17 00:00:00 2001 From: Charles Martin Date: Tue, 11 Jun 2024 16:53:51 +1000 Subject: [PATCH] included prediction speed experiment in the main repo --- .gitignore | 1 + impsy/impsy.py | 3 +- impsy/tests.py | 69 +++++++++++++++++++++++++++- utils/test_prediction_speed.py | 84 ---------------------------------- 4 files changed, 71 insertions(+), 86 deletions(-) delete mode 100644 utils/test_prediction_speed.py diff --git a/.gitignore b/.gitignore index 8056b51..890fecd 100644 --- a/.gitignore +++ b/.gitignore @@ -168,3 +168,4 @@ datasets *.hdf5 *.no *.npz +*.csv diff --git a/impsy/impsy.py b/impsy/impsy.py index 33053cf..d154e39 100644 --- a/impsy/impsy.py +++ b/impsy/impsy.py @@ -5,7 +5,7 @@ from .dataset import dataset from .train import train from .interaction import run -from .tests import test_mdrnn +from .tests import test_mdrnn, prediction_speed_test @click.group() def cli(): @@ -18,5 +18,6 @@ def main(): cli.add_command(train) cli.add_command(run) cli.add_command(test_mdrnn) + cli.add_command(prediction_speed_test) # runs the command line interface cli() \ No newline at end of file diff --git a/impsy/tests.py b/impsy/tests.py index a69df67..cdd9312 100644 --- a/impsy/tests.py +++ b/impsy/tests.py @@ -1,6 +1,10 @@ import click import time from .utils import mdrnn_config +import pandas as pd + + +pd.set_option('display.float_format', lambda x: '%.2f' % x) @click.command(name="test-mdrnn") @@ -25,7 +29,6 @@ def build_network(sess, dimension, units, mixes, layers): n_hidden_units=units, n_mixtures=mixes, layers=layers) - print("MDRNN Loaded.") return net start_build = time.time() @@ -34,3 +37,67 @@ def build_network(sess, dimension, units, mixes, layers): sess = tf.Session() build_network(sess, 4, model_config["units"], model_config["mixes"], model_config["layers"]) print("Done. That took", time.time() - start_build, "seconds.") + + +@click.command(name="test-speed") +def prediction_speed_test(): + """This command runs a speed test experiment with different sized MDRNN models. The output is written to a CSV file.""" + start_import = time.time() + import impsy.mdrnn as mdrnn + import tensorflow.compat.v1 as tf + print("Done. That took", time.time() - start_import, "seconds.") + + def build_network(sess, compute_graph, net_config): + """Build the MDRNN.""" + mdrnn.MODEL_DIR = "./models/" + tf.keras.backend.set_session(sess) + with compute_graph.as_default(): + net = mdrnn.PredictiveMusicMDRNN(mode=mdrnn.NET_MODE_RUN, + dimension=net_config['dimension'], + n_hidden_units=net_config['units'], + n_mixtures=net_config['mixes'], + layers=net_config['layers']) + return net + + def request_rnn_prediction(input_value, net): + """ Accesses a single prediction from the RNN. """ + start = time.time() + output_value = net.generate_touch(input_value) + time_delta = time.time() - start + return output_value, time_delta + + def run_test(tests, config): + times = [] + compute_graph = tf.Graph() + with compute_graph.as_default(): + sess = tf.Session() + net = build_network(sess, compute_graph, config) + for i in range(tests): + ## Predictions. + item = mdrnn.random_sample(out_dim=config['dimension']) + tf.keras.backend.set_session(sess) + with compute_graph.as_default(): + rnn_output, t = request_rnn_prediction(item, net) + out_dict = { + 'time': t, + 'mixes': config['mixes'], + 'layers': config['layers'], + 'units': config['units'], + 'dimension': config['dimension']} + times.append(out_dict) + # clean up + tf.keras.backend.clear_session() + sess.close() + return times + + experiments = [] + mdrnn_units = [64, 128, 256, 512] + dimensions = [2, 3, 4, 5, 6, 7, 8, 9] + for un in mdrnn_units: + for dim in dimensions: + net_config = {"mixes": 5, "layers": 2, "units": un, "dimension": dim} + times = run_test(100, net_config) + experiments.extend(times) + total_experiment = pd.DataFrame.from_records(experiments) + total_experiment.to_csv("total_exp.csv") + print(total_experiment.describe()) diff --git a/utils/test_prediction_speed.py b/utils/test_prediction_speed.py deleted file mode 100644 index 0388b42..0000000 --- a/utils/test_prediction_speed.py +++ /dev/null @@ -1,84 +0,0 @@ -import logging -import time -import datetime -import numpy as np -import pandas as pd - -# Hack to get openMP working annoyingly. -import os -os.environ['KMP_DUPLICATE_LIB_OK']='True' - -print("Importing Keras and MDRNN.") -start_import = time.time() -import empi_mdrnn -import tensorflow as tf -from keras import backend as K -from tensorflow.contrib.training.python.training.hparam import HParams -print("Done. That took", time.time() - start_import, "seconds.") - -def build_network(sess, compute_graph, net_config): - """Build the MDRNN.""" - empi_mdrnn.MODEL_DIR = "./models/" - K.set_session(sess) - with compute_graph.as_default(): - net = empi_mdrnn.PredictiveMusicMDRNN(mode=empi_mdrnn.NET_MODE_RUN, - dimension=net_config.dimension, - n_hidden_units=net_config.units, - n_mixtures=net_config.mixes, - layers=net_config.layers) - #net.pi_temp = net_config.pi_temp - #net.sigma_temp = net_config.sigmatemp - print("MDRNN Loaded.") - return net - - -def request_rnn_prediction(input_value, net): - """ Accesses a single prediction from the RNN. """ - start = time.time() - output_value = net.generate_touch(input_value) - time_delta = time.time() - start - #print("Prediction took:", time_delta) - return output_value, time_delta - - -def run_test(tests, net_config): - times = pd.DataFrame() - compute_graph = tf.Graph() - with compute_graph.as_default(): - sess = tf.Session() - net = build_network(sess, compute_graph, net_config) - for i in range(tests): - ## Predictions. - item = empi_mdrnn.random_sample(out_dim=net_config.dimension) - K.set_session(sess) - with compute_graph.as_default(): - rnn_output, t = request_rnn_prediction(item, net) - out_dict = { - 'time': t, - 'mixes': net_config.mixes, - 'layers': net_config.layers, - 'units': net_config.units, - 'dimension': net_config.dimension} - times = times.append(out_dict, ignore_index=True) - # clean up - K.clear_session() - sess.close() - return times - - -if __name__ == "__main__": - experiment_frames = [] - # hparams = HParams(mixes=5, layers=2, units=64, dimension=2) - mdrnn_units = [64, 128, 256, 512] - dimensions = [2, 3, 4, 5, 6, 7, 8, 9] - for un in mdrnn_units: - for dim in dimensions: - hparams = HParams(mixes=5, layers=2, units=un, dimension=dim) - times = run_test(100, hparams) - experiment_frames.append(times) - total_experiment = pd.concat(experiment_frames, ignore_index=True) - total_experiment.to_csv("total_exp.csv") - print(total_experiment.describe()) - - -# sysctl -n machdep.cpu.brand_string