From eff47f7d61012913fbf63992658c382360e80714 Mon Sep 17 00:00:00 2001 From: Daniel Konegen Date: Wed, 22 Sep 2021 15:11:10 +0200 Subject: [PATCH 1/5] MNIST examples --- .../2_MNIST_on_PC/1_MNIST_train/.gitignore | 6 + examples/2_MNIST_on_PC/1_MNIST_train/LICENSE | 674 +++++++++ .../1_MNIST_train/MNIST_train.cbp | 185 +++ .../1_MNIST_train/MNIST_train.depend | 1259 +++++++++++++++++ .../1_MNIST_train/MNIST_train.layout | 10 + .../2_MNIST_on_PC/1_MNIST_train/keywords.txt | 217 +++ .../1_MNIST_train/src/MNIST_to_cc.py | 81 ++ .../2_MNIST_on_PC/1_MNIST_train/src/aifes.h | 113 ++ .../base/aialgo/aialgo_sequential_inference.c | 224 +++ .../base/aialgo/aialgo_sequential_inference.h | 145 ++ .../base/aialgo/aialgo_sequential_training.c | 299 ++++ .../base/aialgo/aialgo_sequential_training.h | 148 ++ .../src/basic/base/ailayer/ailayer_dense.c | 246 ++++ .../src/basic/base/ailayer/ailayer_dense.h | 296 ++++ .../src/basic/base/ailayer/ailayer_elu.c | 113 ++ .../src/basic/base/ailayer/ailayer_elu.h | 200 +++ .../src/basic/base/ailayer/ailayer_input.c | 99 ++ .../src/basic/base/ailayer/ailayer_input.h | 120 ++ .../basic/base/ailayer/ailayer_leaky_relu.c | 112 ++ .../basic/base/ailayer/ailayer_leaky_relu.h | 200 +++ .../src/basic/base/ailayer/ailayer_relu.c | 110 ++ .../src/basic/base/ailayer/ailayer_relu.h | 187 +++ .../src/basic/base/ailayer/ailayer_sigmoid.c | 122 ++ .../src/basic/base/ailayer/ailayer_sigmoid.h | 182 +++ .../src/basic/base/ailayer/ailayer_softmax.c | 95 ++ .../src/basic/base/ailayer/ailayer_softmax.h | 140 ++ .../src/basic/base/ailayer/ailayer_softsign.c | 120 ++ .../src/basic/base/ailayer/ailayer_softsign.h | 183 +++ .../src/basic/base/ailayer/ailayer_tanh.c | 123 ++ .../src/basic/base/ailayer/ailayer_tanh.h | 182 +++ .../src/basic/base/ailayer/ailayer_template.c | 211 +++ .../src/basic/base/ailayer/ailayer_template.h | 131 ++ .../basic/base/ailoss/ailoss_crossentropy.c | 110 ++ .../basic/base/ailoss/ailoss_crossentropy.h | 205 +++ .../src/basic/base/ailoss/ailoss_mse.c | 99 ++ .../src/basic/base/ailoss/ailoss_mse.h | 162 +++ .../src/basic/base/aimath/aimath_basic.c | 78 + .../src/basic/base/aimath/aimath_basic.h | 123 ++ .../src/basic/base/aimath/aimath_f32.c | 91 ++ .../src/basic/base/aimath/aimath_f32.h | 134 ++ .../src/basic/base/aiopti/aiopti_adam.c | 149 ++ .../src/basic/base/aiopti/aiopti_adam.h | 295 ++++ .../src/basic/base/aiopti/aiopti_sgd.c | 158 +++ .../src/basic/base/aiopti/aiopti_sgd.h | 272 ++++ .../basic/cmsis/ailayer/ailayer_dense_cmsis.c | 49 + .../basic/cmsis/ailayer/ailayer_dense_cmsis.h | 77 + .../src/basic/cmsis/aimath/aimath_f32_cmsis.c | 104 ++ .../src/basic/cmsis/aimath/aimath_f32_cmsis.h | 130 ++ .../default/ailayer/ailayer_dense_default.c | 39 + .../default/ailayer/ailayer_dense_default.h | 69 + .../default/ailayer/ailayer_elu_default.c | 40 + .../default/ailayer/ailayer_elu_default.h | 66 + .../default/ailayer/ailayer_input_default.c | 32 + .../default/ailayer/ailayer_input_default.h | 59 + .../ailayer/ailayer_leaky_relu_default.c | 40 + .../ailayer/ailayer_leaky_relu_default.h | 68 + .../default/ailayer/ailayer_relu_default.c | 39 + .../default/ailayer/ailayer_relu_default.h | 56 + .../default/ailayer/ailayer_sigmoid_default.c | 64 + .../default/ailayer/ailayer_sigmoid_default.h | 73 + .../default/ailayer/ailayer_softmax_default.c | 35 + .../default/ailayer/ailayer_softmax_default.h | 56 + .../ailayer/ailayer_softsign_default.c | 39 + .../ailayer/ailayer_softsign_default.h | 57 + .../default/ailayer/ailayer_tanh_default.c | 67 + .../default/ailayer/ailayer_tanh_default.h | 70 + .../ailoss/ailoss_crossentropy_default.c | 48 + .../ailoss/ailoss_crossentropy_default.h | 73 + .../basic/default/ailoss/ailoss_mse_default.c | 36 + .../basic/default/ailoss/ailoss_mse_default.h | 58 + .../basic/default/aimath/aimath_f32_default.c | 551 ++++++++ .../basic/default/aimath/aimath_f32_default.h | 1225 ++++++++++++++++ .../default/aiopti/aiopti_adam_default.c | 88 ++ .../default/aiopti/aiopti_adam_default.h | 137 ++ .../basic/default/aiopti/aiopti_sgd_default.c | 58 + .../basic/default/aiopti/aiopti_sgd_default.h | 75 + .../1_MNIST_train/src/core/aifes_core.h | 466 ++++++ .../1_MNIST_train/src/core/aifes_math.h | 107 ++ .../2_MNIST_on_PC/1_MNIST_train/src/main.c | 337 +++++ .../2_MNIST_inference_keras/.gitignore | 5 + .../2_MNIST_inference_keras/LICENSE | 674 +++++++++ .../MNIST_inference_keras.cbp | 187 +++ .../MNIST_inference_keras.depend | 936 ++++++++++++ .../MNIST_inference_keras.layout | 10 + .../2_MNIST_inference_keras/keywords.txt | 217 +++ .../src/MNIST_to_cc.py | 81 ++ .../src/MNIST_weights.py | 98 ++ .../2_MNIST_inference_keras/src/aifes.h | 113 ++ .../base/aialgo/aialgo_sequential_inference.c | 224 +++ .../base/aialgo/aialgo_sequential_inference.h | 145 ++ .../base/aialgo/aialgo_sequential_training.c | 299 ++++ .../base/aialgo/aialgo_sequential_training.h | 148 ++ .../src/basic/base/ailayer/ailayer_dense.c | 246 ++++ .../src/basic/base/ailayer/ailayer_dense.h | 296 ++++ .../src/basic/base/ailayer/ailayer_elu.c | 113 ++ .../src/basic/base/ailayer/ailayer_elu.h | 200 +++ .../src/basic/base/ailayer/ailayer_input.c | 99 ++ .../src/basic/base/ailayer/ailayer_input.h | 120 ++ .../basic/base/ailayer/ailayer_leaky_relu.c | 112 ++ .../basic/base/ailayer/ailayer_leaky_relu.h | 200 +++ .../src/basic/base/ailayer/ailayer_relu.c | 110 ++ .../src/basic/base/ailayer/ailayer_relu.h | 187 +++ .../src/basic/base/ailayer/ailayer_sigmoid.c | 122 ++ .../src/basic/base/ailayer/ailayer_sigmoid.h | 182 +++ .../src/basic/base/ailayer/ailayer_softmax.c | 95 ++ .../src/basic/base/ailayer/ailayer_softmax.h | 140 ++ .../src/basic/base/ailayer/ailayer_softsign.c | 120 ++ .../src/basic/base/ailayer/ailayer_softsign.h | 183 +++ .../src/basic/base/ailayer/ailayer_tanh.c | 123 ++ .../src/basic/base/ailayer/ailayer_tanh.h | 182 +++ .../src/basic/base/ailayer/ailayer_template.c | 211 +++ .../src/basic/base/ailayer/ailayer_template.h | 131 ++ .../basic/base/ailoss/ailoss_crossentropy.c | 110 ++ .../basic/base/ailoss/ailoss_crossentropy.h | 205 +++ .../src/basic/base/ailoss/ailoss_mse.c | 99 ++ .../src/basic/base/ailoss/ailoss_mse.h | 162 +++ .../src/basic/base/aimath/aimath_basic.c | 78 + .../src/basic/base/aimath/aimath_basic.h | 123 ++ .../src/basic/base/aimath/aimath_f32.c | 91 ++ .../src/basic/base/aimath/aimath_f32.h | 134 ++ .../src/basic/base/aiopti/aiopti_adam.c | 149 ++ .../src/basic/base/aiopti/aiopti_adam.h | 295 ++++ .../src/basic/base/aiopti/aiopti_sgd.c | 158 +++ .../src/basic/base/aiopti/aiopti_sgd.h | 272 ++++ .../basic/cmsis/ailayer/ailayer_dense_cmsis.c | 49 + .../basic/cmsis/ailayer/ailayer_dense_cmsis.h | 77 + .../src/basic/cmsis/aimath/aimath_f32_cmsis.c | 104 ++ .../src/basic/cmsis/aimath/aimath_f32_cmsis.h | 130 ++ .../default/ailayer/ailayer_dense_default.c | 39 + .../default/ailayer/ailayer_dense_default.h | 69 + .../default/ailayer/ailayer_elu_default.c | 40 + .../default/ailayer/ailayer_elu_default.h | 66 + .../default/ailayer/ailayer_input_default.c | 32 + .../default/ailayer/ailayer_input_default.h | 59 + .../ailayer/ailayer_leaky_relu_default.c | 40 + .../ailayer/ailayer_leaky_relu_default.h | 68 + .../default/ailayer/ailayer_relu_default.c | 39 + .../default/ailayer/ailayer_relu_default.h | 56 + .../default/ailayer/ailayer_sigmoid_default.c | 64 + .../default/ailayer/ailayer_sigmoid_default.h | 73 + .../default/ailayer/ailayer_softmax_default.c | 35 + .../default/ailayer/ailayer_softmax_default.h | 56 + .../ailayer/ailayer_softsign_default.c | 39 + .../ailayer/ailayer_softsign_default.h | 57 + .../default/ailayer/ailayer_tanh_default.c | 67 + .../default/ailayer/ailayer_tanh_default.h | 70 + .../ailoss/ailoss_crossentropy_default.c | 48 + .../ailoss/ailoss_crossentropy_default.h | 73 + .../basic/default/ailoss/ailoss_mse_default.c | 36 + .../basic/default/ailoss/ailoss_mse_default.h | 58 + .../basic/default/aimath/aimath_f32_default.c | 551 ++++++++ .../basic/default/aimath/aimath_f32_default.h | 1225 ++++++++++++++++ .../default/aiopti/aiopti_adam_default.c | 88 ++ .../default/aiopti/aiopti_adam_default.h | 137 ++ .../basic/default/aiopti/aiopti_sgd_default.c | 58 + .../basic/default/aiopti/aiopti_sgd_default.h | 75 + .../src/core/aifes_core.h | 466 ++++++ .../src/core/aifes_math.h | 107 ++ .../2_MNIST_inference_keras/src/main.c | 198 +++ .../3_MNIST_retrain_keras/.gitignore | 7 + .../3_MNIST_retrain_keras/LICENSE | 674 +++++++++ .../MNIST_retrain_keras.cbp | 186 +++ .../MNIST_retrain_keras.depend | 938 ++++++++++++ .../MNIST_retrain_keras.layout | 15 + .../3_MNIST_retrain_keras/keywords.txt | 217 +++ .../3_MNIST_retrain_keras/src/MNIST_to_cc.py | 81 ++ .../src/MNIST_weights.py | 98 ++ .../3_MNIST_retrain_keras/src/aifes.h | 113 ++ .../base/aialgo/aialgo_sequential_inference.c | 224 +++ .../base/aialgo/aialgo_sequential_inference.h | 145 ++ .../base/aialgo/aialgo_sequential_training.c | 299 ++++ .../base/aialgo/aialgo_sequential_training.h | 148 ++ .../src/basic/base/ailayer/ailayer_dense.c | 246 ++++ .../src/basic/base/ailayer/ailayer_dense.h | 296 ++++ .../src/basic/base/ailayer/ailayer_elu.c | 113 ++ .../src/basic/base/ailayer/ailayer_elu.h | 200 +++ .../src/basic/base/ailayer/ailayer_input.c | 99 ++ .../src/basic/base/ailayer/ailayer_input.h | 120 ++ .../basic/base/ailayer/ailayer_leaky_relu.c | 112 ++ .../basic/base/ailayer/ailayer_leaky_relu.h | 200 +++ .../src/basic/base/ailayer/ailayer_relu.c | 110 ++ .../src/basic/base/ailayer/ailayer_relu.h | 187 +++ .../src/basic/base/ailayer/ailayer_sigmoid.c | 122 ++ .../src/basic/base/ailayer/ailayer_sigmoid.h | 182 +++ .../src/basic/base/ailayer/ailayer_softmax.c | 95 ++ .../src/basic/base/ailayer/ailayer_softmax.h | 140 ++ .../src/basic/base/ailayer/ailayer_softsign.c | 120 ++ .../src/basic/base/ailayer/ailayer_softsign.h | 183 +++ .../src/basic/base/ailayer/ailayer_tanh.c | 123 ++ .../src/basic/base/ailayer/ailayer_tanh.h | 182 +++ .../src/basic/base/ailayer/ailayer_template.c | 211 +++ .../src/basic/base/ailayer/ailayer_template.h | 131 ++ .../basic/base/ailoss/ailoss_crossentropy.c | 110 ++ .../basic/base/ailoss/ailoss_crossentropy.h | 205 +++ .../src/basic/base/ailoss/ailoss_mse.c | 99 ++ .../src/basic/base/ailoss/ailoss_mse.h | 162 +++ .../src/basic/base/aimath/aimath_basic.c | 78 + .../src/basic/base/aimath/aimath_basic.h | 123 ++ .../src/basic/base/aimath/aimath_f32.c | 91 ++ .../src/basic/base/aimath/aimath_f32.h | 134 ++ .../src/basic/base/aiopti/aiopti_adam.c | 149 ++ .../src/basic/base/aiopti/aiopti_adam.h | 295 ++++ .../src/basic/base/aiopti/aiopti_sgd.c | 158 +++ .../src/basic/base/aiopti/aiopti_sgd.h | 272 ++++ .../basic/cmsis/ailayer/ailayer_dense_cmsis.c | 49 + .../basic/cmsis/ailayer/ailayer_dense_cmsis.h | 77 + .../src/basic/cmsis/aimath/aimath_f32_cmsis.c | 104 ++ .../src/basic/cmsis/aimath/aimath_f32_cmsis.h | 130 ++ .../default/ailayer/ailayer_dense_default.c | 39 + .../default/ailayer/ailayer_dense_default.h | 69 + .../default/ailayer/ailayer_elu_default.c | 40 + .../default/ailayer/ailayer_elu_default.h | 66 + .../default/ailayer/ailayer_input_default.c | 32 + .../default/ailayer/ailayer_input_default.h | 59 + .../ailayer/ailayer_leaky_relu_default.c | 40 + .../ailayer/ailayer_leaky_relu_default.h | 68 + .../default/ailayer/ailayer_relu_default.c | 39 + .../default/ailayer/ailayer_relu_default.h | 56 + .../default/ailayer/ailayer_sigmoid_default.c | 64 + .../default/ailayer/ailayer_sigmoid_default.h | 73 + .../default/ailayer/ailayer_softmax_default.c | 35 + .../default/ailayer/ailayer_softmax_default.h | 56 + .../ailayer/ailayer_softsign_default.c | 39 + .../ailayer/ailayer_softsign_default.h | 57 + .../default/ailayer/ailayer_tanh_default.c | 67 + .../default/ailayer/ailayer_tanh_default.h | 70 + .../ailoss/ailoss_crossentropy_default.c | 48 + .../ailoss/ailoss_crossentropy_default.h | 73 + .../basic/default/ailoss/ailoss_mse_default.c | 36 + .../basic/default/ailoss/ailoss_mse_default.h | 58 + .../basic/default/aimath/aimath_f32_default.c | 551 ++++++++ .../basic/default/aimath/aimath_f32_default.h | 1225 ++++++++++++++++ .../default/aiopti/aiopti_adam_default.c | 88 ++ .../default/aiopti/aiopti_adam_default.h | 137 ++ .../basic/default/aiopti/aiopti_sgd_default.c | 58 + .../basic/default/aiopti/aiopti_sgd_default.h | 75 + .../src/core/aifes_core.h | 466 ++++++ .../src/core/aifes_math.h | 107 ++ .../3_MNIST_retrain_keras/src/main.c | 324 +++++ 239 files changed, 38399 insertions(+) create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/.gitignore create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/LICENSE create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/MNIST_train.cbp create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/MNIST_train.depend create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/MNIST_train.layout create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/keywords.txt create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/MNIST_to_cc.py create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/aifes.h create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aialgo/aialgo_sequential_inference.c create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aialgo/aialgo_sequential_inference.h create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aialgo/aialgo_sequential_training.c create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aialgo/aialgo_sequential_training.h create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_dense.c create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_dense.h create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_elu.c create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_elu.h create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_input.c create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_input.h create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_leaky_relu.c create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_leaky_relu.h create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_relu.c create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_relu.h create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_sigmoid.c create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_sigmoid.h create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_softmax.c create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_softmax.h create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_softsign.c create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_softsign.h create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_tanh.c create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_tanh.h create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_template.c create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_template.h create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailoss/ailoss_crossentropy.c create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailoss/ailoss_crossentropy.h create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailoss/ailoss_mse.c create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailoss/ailoss_mse.h create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aimath/aimath_basic.c create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aimath/aimath_basic.h create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aimath/aimath_f32.c create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aimath/aimath_f32.h create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aiopti/aiopti_adam.c create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aiopti/aiopti_adam.h create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aiopti/aiopti_sgd.c create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aiopti/aiopti_sgd.h create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/cmsis/ailayer/ailayer_dense_cmsis.c create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/cmsis/ailayer/ailayer_dense_cmsis.h create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/cmsis/aimath/aimath_f32_cmsis.c create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/cmsis/aimath/aimath_f32_cmsis.h create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_dense_default.c create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_dense_default.h create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_elu_default.c create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_elu_default.h create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_input_default.c create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_input_default.h create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_leaky_relu_default.c create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_leaky_relu_default.h create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_relu_default.c create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_relu_default.h create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_sigmoid_default.c create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_sigmoid_default.h create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_softmax_default.c create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_softmax_default.h create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_softsign_default.c create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_softsign_default.h create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_tanh_default.c create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_tanh_default.h create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailoss/ailoss_crossentropy_default.c create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailoss/ailoss_crossentropy_default.h create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailoss/ailoss_mse_default.c create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailoss/ailoss_mse_default.h create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/aimath/aimath_f32_default.c create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/aimath/aimath_f32_default.h create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/aiopti/aiopti_adam_default.c create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/aiopti/aiopti_adam_default.h create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/aiopti/aiopti_sgd_default.c create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/aiopti/aiopti_sgd_default.h create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/core/aifes_core.h create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/core/aifes_math.h create mode 100644 examples/2_MNIST_on_PC/1_MNIST_train/src/main.c create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/.gitignore create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/LICENSE create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/MNIST_inference_keras.cbp create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/MNIST_inference_keras.depend create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/MNIST_inference_keras.layout create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/keywords.txt create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/MNIST_to_cc.py create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/MNIST_weights.py create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/aifes.h create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aialgo/aialgo_sequential_inference.c create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aialgo/aialgo_sequential_inference.h create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aialgo/aialgo_sequential_training.c create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aialgo/aialgo_sequential_training.h create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_dense.c create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_dense.h create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_elu.c create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_elu.h create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_input.c create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_input.h create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_leaky_relu.c create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_leaky_relu.h create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_relu.c create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_relu.h create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_sigmoid.c create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_sigmoid.h create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_softmax.c create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_softmax.h create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_softsign.c create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_softsign.h create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_tanh.c create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_tanh.h create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_template.c create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_template.h create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailoss/ailoss_crossentropy.c create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailoss/ailoss_crossentropy.h create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailoss/ailoss_mse.c create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailoss/ailoss_mse.h create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aimath/aimath_basic.c create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aimath/aimath_basic.h create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aimath/aimath_f32.c create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aimath/aimath_f32.h create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aiopti/aiopti_adam.c create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aiopti/aiopti_adam.h create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aiopti/aiopti_sgd.c create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aiopti/aiopti_sgd.h create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/cmsis/ailayer/ailayer_dense_cmsis.c create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/cmsis/ailayer/ailayer_dense_cmsis.h create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/cmsis/aimath/aimath_f32_cmsis.c create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/cmsis/aimath/aimath_f32_cmsis.h create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_dense_default.c create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_dense_default.h create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_elu_default.c create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_elu_default.h create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_input_default.c create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_input_default.h create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_leaky_relu_default.c create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_leaky_relu_default.h create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_relu_default.c create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_relu_default.h create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_sigmoid_default.c create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_sigmoid_default.h create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_softmax_default.c create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_softmax_default.h create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_softsign_default.c create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_softsign_default.h create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_tanh_default.c create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_tanh_default.h create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailoss/ailoss_crossentropy_default.c create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailoss/ailoss_crossentropy_default.h create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailoss/ailoss_mse_default.c create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailoss/ailoss_mse_default.h create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/aimath/aimath_f32_default.c create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/aimath/aimath_f32_default.h create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/aiopti/aiopti_adam_default.c create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/aiopti/aiopti_adam_default.h create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/aiopti/aiopti_sgd_default.c create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/aiopti/aiopti_sgd_default.h create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/core/aifes_core.h create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/core/aifes_math.h create mode 100644 examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/main.c create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/.gitignore create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/LICENSE create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/MNIST_retrain_keras.cbp create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/MNIST_retrain_keras.depend create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/MNIST_retrain_keras.layout create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/keywords.txt create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/MNIST_to_cc.py create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/MNIST_weights.py create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/aifes.h create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aialgo/aialgo_sequential_inference.c create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aialgo/aialgo_sequential_inference.h create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aialgo/aialgo_sequential_training.c create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aialgo/aialgo_sequential_training.h create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_dense.c create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_dense.h create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_elu.c create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_elu.h create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_input.c create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_input.h create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_leaky_relu.c create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_leaky_relu.h create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_relu.c create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_relu.h create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_sigmoid.c create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_sigmoid.h create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_softmax.c create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_softmax.h create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_softsign.c create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_softsign.h create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_tanh.c create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_tanh.h create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_template.c create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_template.h create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailoss/ailoss_crossentropy.c create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailoss/ailoss_crossentropy.h create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailoss/ailoss_mse.c create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailoss/ailoss_mse.h create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aimath/aimath_basic.c create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aimath/aimath_basic.h create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aimath/aimath_f32.c create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aimath/aimath_f32.h create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aiopti/aiopti_adam.c create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aiopti/aiopti_adam.h create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aiopti/aiopti_sgd.c create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aiopti/aiopti_sgd.h create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/cmsis/ailayer/ailayer_dense_cmsis.c create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/cmsis/ailayer/ailayer_dense_cmsis.h create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/cmsis/aimath/aimath_f32_cmsis.c create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/cmsis/aimath/aimath_f32_cmsis.h create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_dense_default.c create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_dense_default.h create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_elu_default.c create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_elu_default.h create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_input_default.c create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_input_default.h create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_leaky_relu_default.c create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_leaky_relu_default.h create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_relu_default.c create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_relu_default.h create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_sigmoid_default.c create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_sigmoid_default.h create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_softmax_default.c create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_softmax_default.h create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_softsign_default.c create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_softsign_default.h create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_tanh_default.c create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_tanh_default.h create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailoss/ailoss_crossentropy_default.c create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailoss/ailoss_crossentropy_default.h create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailoss/ailoss_mse_default.c create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailoss/ailoss_mse_default.h create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/aimath/aimath_f32_default.c create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/aimath/aimath_f32_default.h create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/aiopti/aiopti_adam_default.c create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/aiopti/aiopti_adam_default.h create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/aiopti/aiopti_sgd_default.c create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/aiopti/aiopti_sgd_default.h create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/core/aifes_core.h create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/core/aifes_math.h create mode 100644 examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/main.c diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/.gitignore b/examples/2_MNIST_on_PC/1_MNIST_train/.gitignore new file mode 100644 index 0000000..1969b93 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/.gitignore @@ -0,0 +1,6 @@ +bin/ +obj/ +/src/MNIST_training_data.h +/src/MNIST_training_data_label.h +/src/MNIST_test_data.h +/src/MNIST_test_data_label.h \ No newline at end of file diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/LICENSE b/examples/2_MNIST_on_PC/1_MNIST_train/LICENSE new file mode 100644 index 0000000..f288702 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/MNIST_train.cbp b/examples/2_MNIST_on_PC/1_MNIST_train/MNIST_train.cbp new file mode 100644 index 0000000..109f6f3 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/MNIST_train.cbp @@ -0,0 +1,185 @@ + + + + + + diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/MNIST_train.depend b/examples/2_MNIST_on_PC/1_MNIST_train/MNIST_train.depend new file mode 100644 index 0000000..0ece4a1 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/MNIST_train.depend @@ -0,0 +1,1259 @@ +# depslib dependency file v1.0 +1631882463 source:c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\base\aialgo\aialgo_sequential_inference.c + "basic/base/aialgo/aialgo_sequential_inference.h" + "basic/default/aimath/aimath_f32_default.h" + + + + +1628682756 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\base\aialgo\aialgo_sequential_inference.h + "core/aifes_core.h" + "core/aifes_math.h" + "basic/base/aimath/aimath_basic.h" + "basic/default/ailayer/ailayer_dense_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\core\aifes_core.h + "aifes_math.h" + +1628682756 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\core\aifes_math.h + + + + + +1628682756 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\base\aimath\aimath_basic.h + "core/aifes_math.h" + +1628682756 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\default\ailayer\ailayer_dense_default.h + "basic/base/ailayer/ailayer_dense.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\base\ailayer\ailayer_dense.h + "core/aifes_core.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\default\aimath\aimath_f32_default.h + + + + + "basic/base/aimath/aimath_f32.h" + +1628682756 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\base\aimath\aimath_f32.h + "core/aifes_math.h" + "basic/base/aimath/aimath_basic.h" + +1632130252 source:c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\base\aialgo\aialgo_sequential_training.c + "basic/base/aialgo/aialgo_sequential_training.h" + "basic/base/aialgo/aialgo_sequential_inference.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\base\aialgo\aialgo_sequential_training.h + "core/aifes_core.h" + "core/aifes_math.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\base\ailayer\ailayer_dense.c + "basic/base/ailayer/ailayer_dense.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\base\ailayer\ailayer_elu.c + "basic/base/ailayer/ailayer_elu.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\base\ailayer\ailayer_elu.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\base\ailayer\ailayer_input.c + "basic/base/ailayer/ailayer_input.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\base\ailayer\ailayer_input.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\base\ailayer\ailayer_leaky_relu.c + "basic/base/ailayer/ailayer_leaky_relu.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\base\ailayer\ailayer_leaky_relu.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\base\ailayer\ailayer_relu.c + "basic/base/ailayer/ailayer_relu.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\base\ailayer\ailayer_relu.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\base\ailayer\ailayer_sigmoid.c + "basic/base/ailayer/ailayer_sigmoid.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\base\ailayer\ailayer_sigmoid.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\base\ailayer\ailayer_softmax.c + "basic/base/ailayer/ailayer_softmax.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\base\ailayer\ailayer_softmax.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\base\ailayer\ailayer_softsign.c + "basic/base/ailayer/ailayer_softsign.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\base\ailayer\ailayer_softsign.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\base\ailayer\ailayer_tanh.c + "basic/base/ailayer/ailayer_tanh.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\base\ailayer\ailayer_tanh.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\base\ailayer\ailayer_template.c + "basic/base/ailayer/ailayer_template.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\base\ailayer\ailayer_template.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\base\ailoss\ailoss_crossentropy.c + "basic/base/ailoss/ailoss_crossentropy.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\base\ailoss\ailoss_crossentropy.h + "core/aifes_core.h" + "basic/base/ailayer/ailayer_softmax.h" + "basic/base/ailayer/ailayer_sigmoid.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\base\ailoss\ailoss_mse.c + "basic/base/ailoss/ailoss_mse.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\base\ailoss\ailoss_mse.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\base\aimath\aimath_basic.c + "basic/base/aimath/aimath_basic.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\base\aimath\aimath_f32.c + "basic/base/aimath/aimath_f32.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\base\aiopti\aiopti_adam.c + "basic/base/aiopti/aiopti_adam.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\base\aiopti\aiopti_adam.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\base\aiopti\aiopti_sgd.c + "basic/base/aiopti/aiopti_sgd.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\base\aiopti\aiopti_sgd.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\cmsis\ailayer\ailayer_dense_cmsis.c + "basic/default/ailayer/ailayer_dense_default.h" + "../../../aifes.h" + "basic/cmsis/ailayer/ailayer_dense_cmsis.h" + "basic/cmsis/aimath/aimath_f32_cmsis.h" + +1628682756 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\aifes.h + "core/aifes_math.h" + "core/aifes_core.h" + "basic/base/aimath/aimath_f32.h" + "basic/base/aimath/aimath_basic.h" + "basic/base/ailayer/ailayer_dense.h" + "basic/base/ailayer/ailayer_input.h" + "basic/base/ailayer/ailayer_relu.h" + "basic/base/ailayer/ailayer_leaky_relu.h" + "basic/base/ailayer/ailayer_elu.h" + "basic/base/ailayer/ailayer_sigmoid.h" + "basic/base/ailayer/ailayer_tanh.h" + "basic/base/ailayer/ailayer_softmax.h" + "basic/base/ailayer/ailayer_softsign.h" + "basic/base/ailoss/ailoss_mse.h" + "basic/base/ailoss/ailoss_crossentropy.h" + "basic/base/aiopti/aiopti_sgd.h" + "basic/base/aiopti/aiopti_adam.h" + "basic/default/aimath/aimath_f32_default.h" + "basic/default/ailayer/ailayer_dense_default.h" + "basic/default/ailayer/ailayer_input_default.h" + "basic/default/ailayer/ailayer_relu_default.h" + "basic/default/ailayer/ailayer_leaky_relu_default.h" + "basic/default/ailayer/ailayer_elu_default.h" + "basic/default/ailayer/ailayer_sigmoid_default.h" + "basic/default/ailayer/ailayer_tanh_default.h" + "basic/default/ailayer/ailayer_softmax_default.h" + "basic/default/ailayer/ailayer_softsign_default.h" + "basic/default/ailoss/ailoss_mse_default.h" + "basic/default/ailoss/ailoss_crossentropy_default.h" + "basic/default/aiopti/aiopti_sgd_default.h" + "basic/default/aiopti/aiopti_adam_default.h" + "basic/cmsis/aimath/aimath_f32_cmsis.h" + "basic/cmsis/ailayer/ailayer_dense_cmsis.h" + "basic/base/aialgo/aialgo_sequential_inference.h" + "basic/base/aialgo/aialgo_sequential_training.h" + +1628682756 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\default\ailayer\ailayer_input_default.h + "basic/base/ailayer/ailayer_input.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\default\ailayer\ailayer_relu_default.h + "basic/base/ailayer/ailayer_relu.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\default\ailayer\ailayer_leaky_relu_default.h + "basic/base/ailayer/ailayer_leaky_relu.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\default\ailayer\ailayer_elu_default.h + "basic/base/ailayer/ailayer_elu.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\default\ailayer\ailayer_sigmoid_default.h + "basic/base/ailayer/ailayer_sigmoid.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\default\ailayer\ailayer_tanh_default.h + "basic/base/ailayer/ailayer_tanh.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\default\ailayer\ailayer_softmax_default.h + "basic/base/ailayer/ailayer_softmax.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\default\ailayer\ailayer_softsign_default.h + "basic/base/ailayer/ailayer_softsign.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\default\ailoss\ailoss_mse_default.h + "basic/base/ailoss/ailoss_mse.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\default\ailoss\ailoss_crossentropy_default.h + "basic/base/ailoss/ailoss_crossentropy.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\default\aiopti\aiopti_sgd_default.h + "basic/base/aiopti/aiopti_sgd.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\default\aiopti\aiopti_adam_default.h + "basic/base/aiopti/aiopti_adam.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\cmsis\aimath\aimath_f32_cmsis.h + "basic/base/aimath/aimath_f32.h" + "../../../aifes.h" + +1628682756 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\cmsis\ailayer\ailayer_dense_cmsis.h + "../../../aifes.h" + "basic/base/ailayer/ailayer_dense.h" + "basic/cmsis/aimath/aimath_f32_cmsis.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\cmsis\aimath\aimath_f32_cmsis.c + "basic/cmsis/aimath/aimath_f32_cmsis.h" + "../../../aifes.h" + "arm_math.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\default\ailayer\ailayer_dense_default.c + "basic/default/ailayer/ailayer_dense_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\default\ailayer\ailayer_elu_default.c + "basic/default/ailayer/ailayer_elu_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\default\ailayer\ailayer_input_default.c + "basic/default/ailayer/ailayer_input_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\default\ailayer\ailayer_leaky_relu_default.c + "basic/default/ailayer/ailayer_leaky_relu_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\default\ailayer\ailayer_relu_default.c + "basic/default/ailayer/ailayer_relu_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\default\ailayer\ailayer_sigmoid_default.c + "basic/default/ailayer/ailayer_sigmoid_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\default\ailayer\ailayer_softmax_default.c + "basic/default/ailayer/ailayer_softmax_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\default\ailayer\ailayer_softsign_default.c + "basic/default/ailayer/ailayer_softsign_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\default\ailayer\ailayer_tanh_default.c + "basic/default/ailayer/ailayer_tanh_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\default\ailoss\ailoss_crossentropy_default.c + "basic/default/ailoss/ailoss_crossentropy_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\default\ailoss\ailoss_mse_default.c + "basic/default/ailoss/ailoss_mse_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\default\aimath\aimath_f32_default.c + "basic/default/aimath/aimath_f32_default.h" + + +1628682756 source:c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\default\aiopti\aiopti_adam_default.c + "basic/default/aiopti/aiopti_adam_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\basic\default\aiopti\aiopti_sgd_default.c + "basic/default/aiopti/aiopti_sgd_default.h" + +1632204420 source:c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\main.c + + + + + "aifes.h" + "MNIST_training_data.h" + "MNIST_training_data_label.h" + "MNIST_test_data.h" + "MNIST_test_data_label.h" + +1632137043 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\mnist_training_data.h + +1632137043 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\mnist_training_data_label.h + +1632131983 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\mnist_test_data.h + +1632131983 c:\users\dk100\documents\aifes\mnist\aifes_for_arduino\src\mnist_test_data_label.h + +1631882463 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\base\aialgo\aialgo_sequential_inference.c + "basic/base/aialgo/aialgo_sequential_inference.h" + "basic/default/aimath/aimath_f32_default.h" + + + + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\base\aialgo\aialgo_sequential_inference.h + "core/aifes_core.h" + "core/aifes_math.h" + "basic/base/aimath/aimath_basic.h" + "basic/default/ailayer/ailayer_dense_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\core\aifes_core.h + "aifes_math.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\core\aifes_math.h + + + + + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\base\aimath\aimath_basic.h + "core/aifes_math.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\default\ailayer\ailayer_dense_default.h + "basic/base/ailayer/ailayer_dense.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\base\ailayer\ailayer_dense.h + "core/aifes_core.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\default\aimath\aimath_f32_default.h + + + + + "basic/base/aimath/aimath_f32.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\base\aimath\aimath_f32.h + "core/aifes_math.h" + "basic/base/aimath/aimath_basic.h" + +1632130252 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\base\aialgo\aialgo_sequential_training.c + "basic/base/aialgo/aialgo_sequential_training.h" + "basic/base/aialgo/aialgo_sequential_inference.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\base\aialgo\aialgo_sequential_training.h + "core/aifes_core.h" + "core/aifes_math.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\base\ailayer\ailayer_dense.c + "basic/base/ailayer/ailayer_dense.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\base\ailayer\ailayer_elu.c + "basic/base/ailayer/ailayer_elu.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\base\ailayer\ailayer_elu.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\base\ailayer\ailayer_input.c + "basic/base/ailayer/ailayer_input.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\base\ailayer\ailayer_input.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\base\ailayer\ailayer_leaky_relu.c + "basic/base/ailayer/ailayer_leaky_relu.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\base\ailayer\ailayer_leaky_relu.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\base\ailayer\ailayer_relu.c + "basic/base/ailayer/ailayer_relu.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\base\ailayer\ailayer_relu.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\base\ailayer\ailayer_sigmoid.c + "basic/base/ailayer/ailayer_sigmoid.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\base\ailayer\ailayer_sigmoid.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\base\ailayer\ailayer_softmax.c + "basic/base/ailayer/ailayer_softmax.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\base\ailayer\ailayer_softmax.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\base\ailayer\ailayer_softsign.c + "basic/base/ailayer/ailayer_softsign.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\base\ailayer\ailayer_softsign.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\base\ailayer\ailayer_tanh.c + "basic/base/ailayer/ailayer_tanh.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\base\ailayer\ailayer_tanh.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\base\ailayer\ailayer_template.c + "basic/base/ailayer/ailayer_template.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\base\ailayer\ailayer_template.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\base\ailoss\ailoss_crossentropy.c + "basic/base/ailoss/ailoss_crossentropy.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\base\ailoss\ailoss_crossentropy.h + "core/aifes_core.h" + "basic/base/ailayer/ailayer_softmax.h" + "basic/base/ailayer/ailayer_sigmoid.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\base\ailoss\ailoss_mse.c + "basic/base/ailoss/ailoss_mse.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\base\ailoss\ailoss_mse.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\base\aimath\aimath_basic.c + "basic/base/aimath/aimath_basic.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\base\aimath\aimath_f32.c + "basic/base/aimath/aimath_f32.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\base\aiopti\aiopti_adam.c + "basic/base/aiopti/aiopti_adam.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\base\aiopti\aiopti_adam.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\base\aiopti\aiopti_sgd.c + "basic/base/aiopti/aiopti_sgd.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\base\aiopti\aiopti_sgd.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\cmsis\ailayer\ailayer_dense_cmsis.c + "basic/default/ailayer/ailayer_dense_default.h" + "../../../aifes.h" + "basic/cmsis/ailayer/ailayer_dense_cmsis.h" + "basic/cmsis/aimath/aimath_f32_cmsis.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\aifes.h + "core/aifes_math.h" + "core/aifes_core.h" + "basic/base/aimath/aimath_f32.h" + "basic/base/aimath/aimath_basic.h" + "basic/base/ailayer/ailayer_dense.h" + "basic/base/ailayer/ailayer_input.h" + "basic/base/ailayer/ailayer_relu.h" + "basic/base/ailayer/ailayer_leaky_relu.h" + "basic/base/ailayer/ailayer_elu.h" + "basic/base/ailayer/ailayer_sigmoid.h" + "basic/base/ailayer/ailayer_tanh.h" + "basic/base/ailayer/ailayer_softmax.h" + "basic/base/ailayer/ailayer_softsign.h" + "basic/base/ailoss/ailoss_mse.h" + "basic/base/ailoss/ailoss_crossentropy.h" + "basic/base/aiopti/aiopti_sgd.h" + "basic/base/aiopti/aiopti_adam.h" + "basic/default/aimath/aimath_f32_default.h" + "basic/default/ailayer/ailayer_dense_default.h" + "basic/default/ailayer/ailayer_input_default.h" + "basic/default/ailayer/ailayer_relu_default.h" + "basic/default/ailayer/ailayer_leaky_relu_default.h" + "basic/default/ailayer/ailayer_elu_default.h" + "basic/default/ailayer/ailayer_sigmoid_default.h" + "basic/default/ailayer/ailayer_tanh_default.h" + "basic/default/ailayer/ailayer_softmax_default.h" + "basic/default/ailayer/ailayer_softsign_default.h" + "basic/default/ailoss/ailoss_mse_default.h" + "basic/default/ailoss/ailoss_crossentropy_default.h" + "basic/default/aiopti/aiopti_sgd_default.h" + "basic/default/aiopti/aiopti_adam_default.h" + "basic/cmsis/aimath/aimath_f32_cmsis.h" + "basic/cmsis/ailayer/ailayer_dense_cmsis.h" + "basic/base/aialgo/aialgo_sequential_inference.h" + "basic/base/aialgo/aialgo_sequential_training.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\default\ailayer\ailayer_input_default.h + "basic/base/ailayer/ailayer_input.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\default\ailayer\ailayer_relu_default.h + "basic/base/ailayer/ailayer_relu.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\default\ailayer\ailayer_leaky_relu_default.h + "basic/base/ailayer/ailayer_leaky_relu.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\default\ailayer\ailayer_elu_default.h + "basic/base/ailayer/ailayer_elu.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\default\ailayer\ailayer_sigmoid_default.h + "basic/base/ailayer/ailayer_sigmoid.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\default\ailayer\ailayer_tanh_default.h + "basic/base/ailayer/ailayer_tanh.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\default\ailayer\ailayer_softmax_default.h + "basic/base/ailayer/ailayer_softmax.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\default\ailayer\ailayer_softsign_default.h + "basic/base/ailayer/ailayer_softsign.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\default\ailoss\ailoss_mse_default.h + "basic/base/ailoss/ailoss_mse.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\default\ailoss\ailoss_crossentropy_default.h + "basic/base/ailoss/ailoss_crossentropy.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\default\aiopti\aiopti_sgd_default.h + "basic/base/aiopti/aiopti_sgd.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\default\aiopti\aiopti_adam_default.h + "basic/base/aiopti/aiopti_adam.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\cmsis\aimath\aimath_f32_cmsis.h + "basic/base/aimath/aimath_f32.h" + "../../../aifes.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\cmsis\ailayer\ailayer_dense_cmsis.h + "../../../aifes.h" + "basic/base/ailayer/ailayer_dense.h" + "basic/cmsis/aimath/aimath_f32_cmsis.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\cmsis\aimath\aimath_f32_cmsis.c + "basic/cmsis/aimath/aimath_f32_cmsis.h" + "../../../aifes.h" + "arm_math.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\default\ailayer\ailayer_dense_default.c + "basic/default/ailayer/ailayer_dense_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\default\ailayer\ailayer_elu_default.c + "basic/default/ailayer/ailayer_elu_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\default\ailayer\ailayer_input_default.c + "basic/default/ailayer/ailayer_input_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\default\ailayer\ailayer_leaky_relu_default.c + "basic/default/ailayer/ailayer_leaky_relu_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\default\ailayer\ailayer_relu_default.c + "basic/default/ailayer/ailayer_relu_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\default\ailayer\ailayer_sigmoid_default.c + "basic/default/ailayer/ailayer_sigmoid_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\default\ailayer\ailayer_softmax_default.c + "basic/default/ailayer/ailayer_softmax_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\default\ailayer\ailayer_softsign_default.c + "basic/default/ailayer/ailayer_softsign_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\default\ailayer\ailayer_tanh_default.c + "basic/default/ailayer/ailayer_tanh_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\default\ailoss\ailoss_crossentropy_default.c + "basic/default/ailoss/ailoss_crossentropy_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\default\ailoss\ailoss_mse_default.c + "basic/default/ailoss/ailoss_mse_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\default\aimath\aimath_f32_default.c + "basic/default/aimath/aimath_f32_default.h" + + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\default\aiopti\aiopti_adam_default.c + "basic/default/aiopti/aiopti_adam_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\mnist\mnist\src\basic\default\aiopti\aiopti_sgd_default.c + "basic/default/aiopti/aiopti_sgd_default.h" + +1631882463 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\base\aialgo\aialgo_sequential_inference.c + "basic/base/aialgo/aialgo_sequential_inference.h" + "basic/default/aimath/aimath_f32_default.h" + + + + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\base\aialgo\aialgo_sequential_inference.h + "core/aifes_core.h" + "core/aifes_math.h" + "basic/base/aimath/aimath_basic.h" + "basic/default/ailayer/ailayer_dense_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\core\aifes_core.h + "aifes_math.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\core\aifes_math.h + + + + + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\base\aimath\aimath_basic.h + "core/aifes_math.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\default\ailayer\ailayer_dense_default.h + "basic/base/ailayer/ailayer_dense.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\base\ailayer\ailayer_dense.h + "core/aifes_core.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\default\aimath\aimath_f32_default.h + + + + + "basic/base/aimath/aimath_f32.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\base\aimath\aimath_f32.h + "core/aifes_math.h" + "basic/base/aimath/aimath_basic.h" + +1632130252 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\base\aialgo\aialgo_sequential_training.c + "basic/base/aialgo/aialgo_sequential_training.h" + "basic/base/aialgo/aialgo_sequential_inference.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\base\aialgo\aialgo_sequential_training.h + "core/aifes_core.h" + "core/aifes_math.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\base\ailayer\ailayer_dense.c + "basic/base/ailayer/ailayer_dense.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\base\ailayer\ailayer_elu.c + "basic/base/ailayer/ailayer_elu.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\base\ailayer\ailayer_elu.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\base\ailayer\ailayer_input.c + "basic/base/ailayer/ailayer_input.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\base\ailayer\ailayer_input.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\base\ailayer\ailayer_leaky_relu.c + "basic/base/ailayer/ailayer_leaky_relu.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\base\ailayer\ailayer_leaky_relu.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\base\ailayer\ailayer_relu.c + "basic/base/ailayer/ailayer_relu.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\base\ailayer\ailayer_relu.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\base\ailayer\ailayer_sigmoid.c + "basic/base/ailayer/ailayer_sigmoid.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\base\ailayer\ailayer_sigmoid.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\base\ailayer\ailayer_softmax.c + "basic/base/ailayer/ailayer_softmax.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\base\ailayer\ailayer_softmax.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\base\ailayer\ailayer_softsign.c + "basic/base/ailayer/ailayer_softsign.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\base\ailayer\ailayer_softsign.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\base\ailayer\ailayer_tanh.c + "basic/base/ailayer/ailayer_tanh.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\base\ailayer\ailayer_tanh.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\base\ailayer\ailayer_template.c + "basic/base/ailayer/ailayer_template.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\base\ailayer\ailayer_template.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\base\ailoss\ailoss_crossentropy.c + "basic/base/ailoss/ailoss_crossentropy.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\base\ailoss\ailoss_crossentropy.h + "core/aifes_core.h" + "basic/base/ailayer/ailayer_softmax.h" + "basic/base/ailayer/ailayer_sigmoid.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\base\ailoss\ailoss_mse.c + "basic/base/ailoss/ailoss_mse.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\base\ailoss\ailoss_mse.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\base\aimath\aimath_basic.c + "basic/base/aimath/aimath_basic.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\base\aimath\aimath_f32.c + "basic/base/aimath/aimath_f32.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\base\aiopti\aiopti_adam.c + "basic/base/aiopti/aiopti_adam.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\base\aiopti\aiopti_adam.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\base\aiopti\aiopti_sgd.c + "basic/base/aiopti/aiopti_sgd.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\base\aiopti\aiopti_sgd.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\cmsis\ailayer\ailayer_dense_cmsis.c + "basic/default/ailayer/ailayer_dense_default.h" + "../../../aifes.h" + "basic/cmsis/ailayer/ailayer_dense_cmsis.h" + "basic/cmsis/aimath/aimath_f32_cmsis.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\aifes.h + "core/aifes_math.h" + "core/aifes_core.h" + "basic/base/aimath/aimath_f32.h" + "basic/base/aimath/aimath_basic.h" + "basic/base/ailayer/ailayer_dense.h" + "basic/base/ailayer/ailayer_input.h" + "basic/base/ailayer/ailayer_relu.h" + "basic/base/ailayer/ailayer_leaky_relu.h" + "basic/base/ailayer/ailayer_elu.h" + "basic/base/ailayer/ailayer_sigmoid.h" + "basic/base/ailayer/ailayer_tanh.h" + "basic/base/ailayer/ailayer_softmax.h" + "basic/base/ailayer/ailayer_softsign.h" + "basic/base/ailoss/ailoss_mse.h" + "basic/base/ailoss/ailoss_crossentropy.h" + "basic/base/aiopti/aiopti_sgd.h" + "basic/base/aiopti/aiopti_adam.h" + "basic/default/aimath/aimath_f32_default.h" + "basic/default/ailayer/ailayer_dense_default.h" + "basic/default/ailayer/ailayer_input_default.h" + "basic/default/ailayer/ailayer_relu_default.h" + "basic/default/ailayer/ailayer_leaky_relu_default.h" + "basic/default/ailayer/ailayer_elu_default.h" + "basic/default/ailayer/ailayer_sigmoid_default.h" + "basic/default/ailayer/ailayer_tanh_default.h" + "basic/default/ailayer/ailayer_softmax_default.h" + "basic/default/ailayer/ailayer_softsign_default.h" + "basic/default/ailoss/ailoss_mse_default.h" + "basic/default/ailoss/ailoss_crossentropy_default.h" + "basic/default/aiopti/aiopti_sgd_default.h" + "basic/default/aiopti/aiopti_adam_default.h" + "basic/cmsis/aimath/aimath_f32_cmsis.h" + "basic/cmsis/ailayer/ailayer_dense_cmsis.h" + "basic/base/aialgo/aialgo_sequential_inference.h" + "basic/base/aialgo/aialgo_sequential_training.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\default\ailayer\ailayer_input_default.h + "basic/base/ailayer/ailayer_input.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\default\ailayer\ailayer_relu_default.h + "basic/base/ailayer/ailayer_relu.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\default\ailayer\ailayer_leaky_relu_default.h + "basic/base/ailayer/ailayer_leaky_relu.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\default\ailayer\ailayer_elu_default.h + "basic/base/ailayer/ailayer_elu.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\default\ailayer\ailayer_sigmoid_default.h + "basic/base/ailayer/ailayer_sigmoid.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\default\ailayer\ailayer_tanh_default.h + "basic/base/ailayer/ailayer_tanh.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\default\ailayer\ailayer_softmax_default.h + "basic/base/ailayer/ailayer_softmax.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\default\ailayer\ailayer_softsign_default.h + "basic/base/ailayer/ailayer_softsign.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\default\ailoss\ailoss_mse_default.h + "basic/base/ailoss/ailoss_mse.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\default\ailoss\ailoss_crossentropy_default.h + "basic/base/ailoss/ailoss_crossentropy.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\default\aiopti\aiopti_sgd_default.h + "basic/base/aiopti/aiopti_sgd.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\default\aiopti\aiopti_adam_default.h + "basic/base/aiopti/aiopti_adam.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\cmsis\aimath\aimath_f32_cmsis.h + "basic/base/aimath/aimath_f32.h" + "../../../aifes.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\cmsis\ailayer\ailayer_dense_cmsis.h + "../../../aifes.h" + "basic/base/ailayer/ailayer_dense.h" + "basic/cmsis/aimath/aimath_f32_cmsis.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\cmsis\aimath\aimath_f32_cmsis.c + "basic/cmsis/aimath/aimath_f32_cmsis.h" + "../../../aifes.h" + "arm_math.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\default\ailayer\ailayer_dense_default.c + "basic/default/ailayer/ailayer_dense_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\default\ailayer\ailayer_elu_default.c + "basic/default/ailayer/ailayer_elu_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\default\ailayer\ailayer_input_default.c + "basic/default/ailayer/ailayer_input_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\default\ailayer\ailayer_leaky_relu_default.c + "basic/default/ailayer/ailayer_leaky_relu_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\default\ailayer\ailayer_relu_default.c + "basic/default/ailayer/ailayer_relu_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\default\ailayer\ailayer_sigmoid_default.c + "basic/default/ailayer/ailayer_sigmoid_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\default\ailayer\ailayer_softmax_default.c + "basic/default/ailayer/ailayer_softmax_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\default\ailayer\ailayer_softsign_default.c + "basic/default/ailayer/ailayer_softsign_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\default\ailayer\ailayer_tanh_default.c + "basic/default/ailayer/ailayer_tanh_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\default\ailoss\ailoss_crossentropy_default.c + "basic/default/ailoss/ailoss_crossentropy_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\default\ailoss\ailoss_mse_default.c + "basic/default/ailoss/ailoss_mse_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\default\aimath\aimath_f32_default.c + "basic/default/aimath/aimath_f32_default.h" + + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\default\aiopti\aiopti_adam_default.c + "basic/default/aiopti/aiopti_adam_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_train\src\basic\default\aiopti\aiopti_sgd_default.c + "basic/default/aiopti/aiopti_sgd_default.h" + +1631882463 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\base\aialgo\aialgo_sequential_inference.c + "basic/base/aialgo/aialgo_sequential_inference.h" + "basic/default/aimath/aimath_f32_default.h" + + + + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\base\aialgo\aialgo_sequential_inference.h + "core/aifes_core.h" + "core/aifes_math.h" + "basic/base/aimath/aimath_basic.h" + "basic/default/ailayer/ailayer_dense_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\core\aifes_core.h + "aifes_math.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\core\aifes_math.h + + + + + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\base\aimath\aimath_basic.h + "core/aifes_math.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\default\ailayer\ailayer_dense_default.h + "basic/base/ailayer/ailayer_dense.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\base\ailayer\ailayer_dense.h + "core/aifes_core.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\default\aimath\aimath_f32_default.h + + + + + "basic/base/aimath/aimath_f32.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\base\aimath\aimath_f32.h + "core/aifes_math.h" + "basic/base/aimath/aimath_basic.h" + +1632130252 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\base\aialgo\aialgo_sequential_training.c + "basic/base/aialgo/aialgo_sequential_training.h" + "basic/base/aialgo/aialgo_sequential_inference.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\base\aialgo\aialgo_sequential_training.h + "core/aifes_core.h" + "core/aifes_math.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\base\ailayer\ailayer_dense.c + "basic/base/ailayer/ailayer_dense.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\base\ailayer\ailayer_elu.c + "basic/base/ailayer/ailayer_elu.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\base\ailayer\ailayer_elu.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\base\ailayer\ailayer_input.c + "basic/base/ailayer/ailayer_input.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\base\ailayer\ailayer_input.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\base\ailayer\ailayer_leaky_relu.c + "basic/base/ailayer/ailayer_leaky_relu.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\base\ailayer\ailayer_leaky_relu.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\base\ailayer\ailayer_relu.c + "basic/base/ailayer/ailayer_relu.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\base\ailayer\ailayer_relu.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\base\ailayer\ailayer_sigmoid.c + "basic/base/ailayer/ailayer_sigmoid.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\base\ailayer\ailayer_sigmoid.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\base\ailayer\ailayer_softmax.c + "basic/base/ailayer/ailayer_softmax.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\base\ailayer\ailayer_softmax.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\base\ailayer\ailayer_softsign.c + "basic/base/ailayer/ailayer_softsign.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\base\ailayer\ailayer_softsign.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\base\ailayer\ailayer_tanh.c + "basic/base/ailayer/ailayer_tanh.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\base\ailayer\ailayer_tanh.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\base\ailayer\ailayer_template.c + "basic/base/ailayer/ailayer_template.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\base\ailayer\ailayer_template.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\base\ailoss\ailoss_crossentropy.c + "basic/base/ailoss/ailoss_crossentropy.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\base\ailoss\ailoss_crossentropy.h + "core/aifes_core.h" + "basic/base/ailayer/ailayer_softmax.h" + "basic/base/ailayer/ailayer_sigmoid.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\base\ailoss\ailoss_mse.c + "basic/base/ailoss/ailoss_mse.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\base\ailoss\ailoss_mse.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\base\aimath\aimath_basic.c + "basic/base/aimath/aimath_basic.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\base\aimath\aimath_f32.c + "basic/base/aimath/aimath_f32.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\base\aiopti\aiopti_adam.c + "basic/base/aiopti/aiopti_adam.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\base\aiopti\aiopti_adam.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\base\aiopti\aiopti_sgd.c + "basic/base/aiopti/aiopti_sgd.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\base\aiopti\aiopti_sgd.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\cmsis\ailayer\ailayer_dense_cmsis.c + "basic/default/ailayer/ailayer_dense_default.h" + "../../../aifes.h" + "basic/cmsis/ailayer/ailayer_dense_cmsis.h" + "basic/cmsis/aimath/aimath_f32_cmsis.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\aifes.h + "core/aifes_math.h" + "core/aifes_core.h" + "basic/base/aimath/aimath_f32.h" + "basic/base/aimath/aimath_basic.h" + "basic/base/ailayer/ailayer_dense.h" + "basic/base/ailayer/ailayer_input.h" + "basic/base/ailayer/ailayer_relu.h" + "basic/base/ailayer/ailayer_leaky_relu.h" + "basic/base/ailayer/ailayer_elu.h" + "basic/base/ailayer/ailayer_sigmoid.h" + "basic/base/ailayer/ailayer_tanh.h" + "basic/base/ailayer/ailayer_softmax.h" + "basic/base/ailayer/ailayer_softsign.h" + "basic/base/ailoss/ailoss_mse.h" + "basic/base/ailoss/ailoss_crossentropy.h" + "basic/base/aiopti/aiopti_sgd.h" + "basic/base/aiopti/aiopti_adam.h" + "basic/default/aimath/aimath_f32_default.h" + "basic/default/ailayer/ailayer_dense_default.h" + "basic/default/ailayer/ailayer_input_default.h" + "basic/default/ailayer/ailayer_relu_default.h" + "basic/default/ailayer/ailayer_leaky_relu_default.h" + "basic/default/ailayer/ailayer_elu_default.h" + "basic/default/ailayer/ailayer_sigmoid_default.h" + "basic/default/ailayer/ailayer_tanh_default.h" + "basic/default/ailayer/ailayer_softmax_default.h" + "basic/default/ailayer/ailayer_softsign_default.h" + "basic/default/ailoss/ailoss_mse_default.h" + "basic/default/ailoss/ailoss_crossentropy_default.h" + "basic/default/aiopti/aiopti_sgd_default.h" + "basic/default/aiopti/aiopti_adam_default.h" + "basic/cmsis/aimath/aimath_f32_cmsis.h" + "basic/cmsis/ailayer/ailayer_dense_cmsis.h" + "basic/base/aialgo/aialgo_sequential_inference.h" + "basic/base/aialgo/aialgo_sequential_training.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\default\ailayer\ailayer_input_default.h + "basic/base/ailayer/ailayer_input.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\default\ailayer\ailayer_relu_default.h + "basic/base/ailayer/ailayer_relu.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\default\ailayer\ailayer_leaky_relu_default.h + "basic/base/ailayer/ailayer_leaky_relu.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\default\ailayer\ailayer_elu_default.h + "basic/base/ailayer/ailayer_elu.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\default\ailayer\ailayer_sigmoid_default.h + "basic/base/ailayer/ailayer_sigmoid.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\default\ailayer\ailayer_tanh_default.h + "basic/base/ailayer/ailayer_tanh.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\default\ailayer\ailayer_softmax_default.h + "basic/base/ailayer/ailayer_softmax.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\default\ailayer\ailayer_softsign_default.h + "basic/base/ailayer/ailayer_softsign.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\default\ailoss\ailoss_mse_default.h + "basic/base/ailoss/ailoss_mse.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\default\ailoss\ailoss_crossentropy_default.h + "basic/base/ailoss/ailoss_crossentropy.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\default\aiopti\aiopti_sgd_default.h + "basic/base/aiopti/aiopti_sgd.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\default\aiopti\aiopti_adam_default.h + "basic/base/aiopti/aiopti_adam.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\cmsis\aimath\aimath_f32_cmsis.h + "basic/base/aimath/aimath_f32.h" + "../../../aifes.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\cmsis\ailayer\ailayer_dense_cmsis.h + "../../../aifes.h" + "basic/base/ailayer/ailayer_dense.h" + "basic/cmsis/aimath/aimath_f32_cmsis.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\cmsis\aimath\aimath_f32_cmsis.c + "basic/cmsis/aimath/aimath_f32_cmsis.h" + "../../../aifes.h" + "arm_math.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\default\ailayer\ailayer_dense_default.c + "basic/default/ailayer/ailayer_dense_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\default\ailayer\ailayer_elu_default.c + "basic/default/ailayer/ailayer_elu_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\default\ailayer\ailayer_input_default.c + "basic/default/ailayer/ailayer_input_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\default\ailayer\ailayer_leaky_relu_default.c + "basic/default/ailayer/ailayer_leaky_relu_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\default\ailayer\ailayer_relu_default.c + "basic/default/ailayer/ailayer_relu_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\default\ailayer\ailayer_sigmoid_default.c + "basic/default/ailayer/ailayer_sigmoid_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\default\ailayer\ailayer_softmax_default.c + "basic/default/ailayer/ailayer_softmax_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\default\ailayer\ailayer_softsign_default.c + "basic/default/ailayer/ailayer_softsign_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\default\ailayer\ailayer_tanh_default.c + "basic/default/ailayer/ailayer_tanh_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\default\ailoss\ailoss_crossentropy_default.c + "basic/default/ailoss/ailoss_crossentropy_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\default\ailoss\ailoss_mse_default.c + "basic/default/ailoss/ailoss_mse_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\default\aimath\aimath_f32_default.c + "basic/default/aimath/aimath_f32_default.h" + + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\default\aiopti\aiopti_adam_default.c + "basic/default/aiopti/aiopti_adam_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\basic\default\aiopti\aiopti_sgd_default.c + "basic/default/aiopti/aiopti_sgd_default.h" + +1632312912 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\main.c + + + + + "aifes.h" + "MNIST_training_data.h" + "MNIST_training_data_label.h" + "MNIST_test_data.h" + "MNIST_test_data_label.h" + +1632313103 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\mnist_training_data.h + +1632313103 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\mnist_training_data_label.h + +1632313114 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\mnist_test_data.h + +1632313114 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\1_mnist_train\src\mnist_test_data_label.h + diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/MNIST_train.layout b/examples/2_MNIST_on_PC/1_MNIST_train/MNIST_train.layout new file mode 100644 index 0000000..19290a1 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/MNIST_train.layout @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/keywords.txt b/examples/2_MNIST_on_PC/1_MNIST_train/keywords.txt new file mode 100644 index 0000000..6d759a9 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/keywords.txt @@ -0,0 +1,217 @@ +####################################### +# Syntax Coloring Map For AIfES for Arduino +####################################### + +####################################### +# Datatypes (KEYWORD1) +####################################### + +aimodel_t KEYWORD1 +ailayer_t KEYWORD1 +ailoss_t KEYWORD1 +aiopti_t KEYWORD1 + +aicore_layertype_t KEYWORD1 +aicore_losstype_t KEYWORD1 +aicore_optitype_t KEYWORD1 + +ailayer_dense_t KEYWORD1 +ailayer_elu_t KEYWORD1 +ailayer_elu_f32_t KEYWORD1 +ailayer_input_t KEYWORD1 +ailayer_leaky_relu_t KEYWORD1 +ailayer_leaky_relu_f32_t KEYWORD1 +ailayer_relu_t KEYWORD1 +ailayer_sigmoid_t KEYWORD1 +ailayer_softmax_t KEYWORD1 +ailayer_softsign_t KEYWORD1 +ailayer_tanh_t KEYWORD1 + +ailoss_crossentropy_t KEYWORD1 +ailoss_mse_t KEYWORD1 + +aiopti_adam_t KEYWORD1 +aiopti_adam_f32_t KEYWORD1 +aiopti_adam_momentums_t KEYWORD1 +aiopti_sgd_t KEYWORD1 +aiopti_sgd_f32_t KEYWORD1 + +aitensor_t KEYWORD1 +####################################### +# Methods and Functions (KEYWORD2) +####################################### + +aialgo_backward_model KEYWORD2 +aialgo_calc_loss_model_f32 KEYWORD2 +aialgo_compile_model KEYWORD2 +aialgo_distribute_parameter_memory KEYWORD2 +aialgo_forward_model KEYWORD2 +aialgo_inference_model KEYWORD2 +aialgo_init_model_for_training KEYWORD2 +aialgo_print_loss_specs KEYWORD2 +aialgo_print_model_structure KEYWORD2 +aialgo_print_optimizer_specs KEYWORD2 +aialgo_schedule_inference_memory KEYWORD2 +aialgo_schedule_training_memory KEYWORD2 +aialgo_sizeof_inference_memory KEYWORD2 +aialgo_sizeof_parameter_memory KEYWORD2 +aialgo_sizeof_training_memory KEYWORD2 +aialgo_train_model KEYWORD2 +aialgo_update_params_model KEYWORD2 +aialgo_zero_gradients_model KEYWORD2 +ailayer_dense KEYWORD2 +ailayer_dense_backward KEYWORD2 +ailayer_dense_calc_result_shape KEYWORD2 +ailayer_dense_f32_cmsis KEYWORD2 +ailayer_dense_f32_default KEYWORD2 +ailayer_dense_forward KEYWORD2 +ailayer_dense_print_specs KEYWORD2 +ailayer_dense_set_paramem KEYWORD2 +ailayer_dense_set_trainmem KEYWORD2 +ailayer_dense_sizeof_paramem KEYWORD2 +ailayer_dense_sizeof_trainmem KEYWORD2 +ailayer_elu KEYWORD2 +ailayer_elu_backward KEYWORD2 +ailayer_elu_calc_result_shape KEYWORD2 +ailayer_elu_f32_default KEYWORD2 +ailayer_elu_forward KEYWORD2 +ailayer_elu_print_specs KEYWORD2 +ailayer_input KEYWORD2 +ailayer_input_backward KEYWORD2 +ailayer_input_calc_result_shape KEYWORD2 +ailayer_input_f32_default KEYWORD2 +ailayer_input_forward KEYWORD2 +ailayer_input_print_specs KEYWORD2 +ailayer_leaky_relu KEYWORD2 +ailayer_leaky_relu_backward KEYWORD2 +ailayer_leaky_relu_calc_result_shape KEYWORD2 +ailayer_leaky_relu_f32_default KEYWORD2 +ailayer_leaky_relu_forward KEYWORD2 +ailayer_leaky_relu_print_specs KEYWORD2 +ailayer_relu KEYWORD2 +ailayer_relu_backward KEYWORD2 +ailayer_relu_calc_result_shape KEYWORD2 +ailayer_relu_f32_default KEYWORD2 +ailayer_relu_forward KEYWORD2 +ailayer_relu_print_specs KEYWORD2 +ailayer_sigmoid KEYWORD2 +ailayer_sigmoid_backward KEYWORD2 +ailayer_sigmoid_calc_result_shape KEYWORD2 +ailayer_sigmoid_f32_default KEYWORD2 +ailayer_sigmoid_forward KEYWORD2 +ailayer_sigmoid_get_result_bound_f32_default KEYWORD2 +ailayer_sigmoid_print_specs KEYWORD2 +ailayer_softmax KEYWORD2 +ailayer_softmax_calc_result_shape KEYWORD2 +ailayer_softmax_f32_default KEYWORD2 +ailayer_softmax_forward KEYWORD2 +ailayer_softmax_print_specs KEYWORD2 +ailayer_softsign KEYWORD2 +ailayer_softsign_backward KEYWORD2 +ailayer_softsign_calc_result_shape KEYWORD2 +ailayer_softsign_f32_default KEYWORD2 +ailayer_softsign_forward KEYWORD2 +ailayer_softsign_print_specs KEYWORD2 +ailayer_tanh KEYWORD2 +ailayer_tanh_backward KEYWORD2 +ailayer_tanh_calc_result_shape KEYWORD2 +ailayer_tanh_f32_default KEYWORD2 +ailayer_tanh_forward KEYWORD2 +ailayer_tanh_get_result_bound_f32_default KEYWORD2 +ailayer_tanh_print_specs KEYWORD2 +ailayer_template KEYWORD2 +ailayer_template_backward KEYWORD2 +ailayer_template_calc_result_shape KEYWORD2 +ailayer_template_forward KEYWORD2 +ailayer_template_print_specs KEYWORD2 +ailayer_template_set_paramem KEYWORD2 +ailayer_template_set_trainmem KEYWORD2 +ailayer_template_sizeof_paramem KEYWORD2 +ailayer_template_sizeof_trainmem KEYWORD2 +ailoss_crossentropy KEYWORD2 +ailoss_crossentropy_calc_delta KEYWORD2 +ailoss_crossentropy_calc_loss KEYWORD2 +ailoss_crossentropy_dummy_backward KEYWORD2 +ailoss_crossentropy_f32_default KEYWORD2 +ailoss_crossentropy_print_specs KEYWORD2 +ailoss_mse KEYWORD2 +ailoss_mse_calc_delta KEYWORD2 +ailoss_mse_calc_loss KEYWORD2 +ailoss_mse_f32_default KEYWORD2 +ailoss_mse_print_specs KEYWORD2 +aimath_f32_cmsis_linear KEYWORD2 +aimath_f32_cmsis_mat_mul KEYWORD2 +aimath_f32_default_binary_crossentropy KEYWORD2 +aimath_f32_default_categorical_crossentropy KEYWORD2 +aimath_f32_default_categorical_crossentropy_sparse8 KEYWORD2 +aimath_f32_default_copy_tensor KEYWORD2 +aimath_f32_default_d_elu KEYWORD2 +aimath_f32_default_d_leaky_relu KEYWORD2 +aimath_f32_default_d_relu KEYWORD2 +aimath_f32_default_d_sigmoid KEYWORD2 +aimath_f32_default_d_softsign KEYWORD2 +aimath_f32_default_d_tanh KEYWORD2 +aimath_f32_default_divide KEYWORD2 +aimath_f32_default_elu KEYWORD2 +aimath_f32_default_expf_fast KEYWORD2 +aimath_f32_default_init_glorot_uniform KEYWORD2 +aimath_f32_default_init_he_uniform KEYWORD2 +aimath_f32_default_init_zeros KEYWORD2 +aimath_f32_default_leaky_relu KEYWORD2 +aimath_f32_default_linear KEYWORD2 +aimath_f32_default_mat_mul KEYWORD2 +aimath_f32_default_max KEYWORD2 +aimath_f32_default_min KEYWORD2 +aimath_f32_default_multiply KEYWORD2 +aimath_f32_default_norm_squared KEYWORD2 +aimath_f32_default_relu KEYWORD2 +aimath_f32_default_scalar_add KEYWORD2 +aimath_f32_default_scalar_mul KEYWORD2 +aimath_f32_default_sigmoid KEYWORD2 +aimath_f32_default_softmax KEYWORD2 +aimath_f32_default_softsign KEYWORD2 +aimath_f32_default_sqrt KEYWORD2 +aimath_f32_default_sum KEYWORD2 +aimath_f32_default_tanh KEYWORD2 +aimath_f32_default_tensor_add KEYWORD2 +aimath_f32_default_tensor_init_uniform KEYWORD2 +aimath_f32_default_tensor_sub KEYWORD2 +aimath_f32_default_tensor_sub_sparse8 KEYWORD2 +aimath_f32_default_transpose_vector KEYWORD2 +aimath_f32_default_zero_tensor KEYWORD2 +aimath_f32_print_aiscalar KEYWORD2 +aimath_f32_print_aitensor KEYWORD2 +aimath_sizeof_dtype KEYWORD2 +aimath_sizeof_tensor KEYWORD2 +aimath_sizeof_tensor_data KEYWORD2 +aimath_sizeof_tensor_params KEYWORD2 +aimath_tensor_elements KEYWORD2 +aimath_transpose_vector KEYWORD2 +aiopti_adam KEYWORD2 +aiopti_adam_f32_default KEYWORD2 +aiopti_adam_f32_default_begin_step KEYWORD2 +aiopti_adam_f32_default_end_step KEYWORD2 +aiopti_adam_init_optimem KEYWORD2 +aiopti_adam_print_specs KEYWORD2 +aiopti_adam_sizeof_optimem KEYWORD2 +aiopti_adam_update_params KEYWORD2 +aiopti_adam_zero_gradients KEYWORD2 +aiopti_sgd KEYWORD2 +aiopti_sgd_f32_default KEYWORD2 +aiopti_sgd_init_optimem_with_momentum KEYWORD2 +aiopti_sgd_init_optimem_without_momentum KEYWORD2 +aiopti_sgd_print_specs KEYWORD2 +aiopti_sgd_sizeof_optimem_with_momentum KEYWORD2 +aiopti_sgd_sizeof_optimem_without_momentum KEYWORD2 +aiopti_sgd_update_params_with_momentum KEYWORD2 +aiopti_sgd_update_params_without_momentum KEYWORD2 +aiopti_sgd_zero_gradients KEYWORD2 + +print_aiscalar KEYWORD2 +print_aitensor KEYWORD2 + +####################################### +# Constants (LITERAL1) +####################################### + +aif32 LITERAL1 diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/MNIST_to_cc.py b/examples/2_MNIST_on_PC/1_MNIST_train/src/MNIST_to_cc.py new file mode 100644 index 0000000..fbfe515 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/MNIST_to_cc.py @@ -0,0 +1,81 @@ +import tensorflow as tf +from tensorflow.keras.utils import to_categorical + + +#Load and preprocess the MNIST data set +(x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data() + +x_train = x_train.astype(float)/255.0 +x_test = x_test.astype(float)/255.0 + +y_train = to_categorical(y_train) +y_test = to_categorical(y_test) + +print(x_train.shape) +print(x_test.shape) + +NUM_TRAINING_DATA = 30000 #Max 60000 +NUM_TEST_DATA = 10000 #Max 10000 + +def generate_train_data(): + ''' + Generates two header files. The first one contains as many training data according to the + number defined above. The second header file contains the labels belonging to the training + data. The training data and labels are both stored in a two-dimensional array. + ''' + + with open("MNIST_training_data.h", "w") as f: + f.write("float MNIST_training_data[" + str(NUM_TRAINING_DATA) + "][784] = {\n") + for i in range(0,NUM_TRAINING_DATA): + if i != 0: + f.write("},\n") + x_train_flatten = x_train[i].flatten() + f.write("{" + str(x_train_flatten[0]) + "f") + for j in range(1,784): + f.write(", " + str(x_train_flatten[j]) + "f") + f.write("}\n};") + + + with open("MNIST_training_data_label.h", "w") as f: + f.write("float MNIST_training_data_label[" + str(NUM_TRAINING_DATA) + "][10] = {\n") + for i in range(0,NUM_TRAINING_DATA): + if i != 0: + f.write("},\n") + f.write("{" + str(y_train[i][0]) + "f") + for j in range(1,10): + f.write(", " + str(y_train[i][j]) + "f") + f.write("}\n};") + + +def generate_test_data(): + ''' + Generates two header files. The first one contains as many test data according to the + number defined above. The second header file contains the labels belonging to the test + data. The test data and labels are both stored in a two-dimensional array. + ''' + + with open("MNIST_test_data.h", "w") as f: + f.write("float MNIST_test_data[" + str(NUM_TEST_DATA) + "][784] = {\n") + for i in range(0,NUM_TEST_DATA): + if i != 0: + f.write("},\n") + x_test_flatten = x_test[i].flatten() + f.write("{" + str(x_test_flatten[0]) + "f") + for j in range(1,784): + f.write(", " + str(x_test_flatten[j]) + "f") + f.write("}\n};") + + + with open("MNIST_test_data_label.h", "w") as f: + f.write("float MNIST_test_data_label[" + str(NUM_TEST_DATA) + "][10] = {\n") + for i in range(0,NUM_TEST_DATA): + if i != 0: + f.write("},\n") + f.write("{" + str(y_test[i][0]) + "f") + for j in range(1,10): + f.write(", " + str(y_test[i][j]) + "f") + f.write("}\n};") + + +generate_train_data() +generate_test_data() \ No newline at end of file diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/aifes.h b/examples/2_MNIST_on_PC/1_MNIST_train/src/aifes.h new file mode 100644 index 0000000..e7af402 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/aifes.h @@ -0,0 +1,113 @@ +/** + * \file basic/aifes.h + * \internal + * \date 23.02.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Include all headers of the aifes 2 - basic module + * \details + */ + +#ifdef __cplusplus +extern "C" { +#endif + + +// Include AIfES core headers +#include "core/aifes_math.h" +#include "core/aifes_core.h" + +// Include the datatypes +#include "basic/base/aimath/aimath_f32.h" + +// Include basic datatype independent math functions +#include "basic/base/aimath/aimath_basic.h" + +// ---------------------------- Module base implementations ----------------------- +// ("abstract" super "classes". A hardware optimized implementation can "inherit" from these modules) + +// Include the layer base implementations +#include "basic/base/ailayer/ailayer_dense.h" +#include "basic/base/ailayer/ailayer_input.h" +#include "basic/base/ailayer/ailayer_relu.h" +#include "basic/base/ailayer/ailayer_leaky_relu.h" +#include "basic/base/ailayer/ailayer_elu.h" +#include "basic/base/ailayer/ailayer_sigmoid.h" +#include "basic/base/ailayer/ailayer_tanh.h" +#include "basic/base/ailayer/ailayer_softmax.h" +#include "basic/base/ailayer/ailayer_softsign.h" + +// Include the loss base implementations +#include "basic/base/ailoss/ailoss_mse.h" +#include "basic/base/ailoss/ailoss_crossentropy.h" + +// Include the optimizer base implementations +#include "basic/base/aiopti/aiopti_sgd.h" +#include "basic/base/aiopti/aiopti_adam.h" + +// ---------------------------- Module default implementations ----------------------- +// (Fallback functions if no hardware optimized implementation available) + +// Include the math in default implementation +#include "basic/default/aimath/aimath_f32_default.h" + +// Include the layers in default implementation +#include "basic/default/ailayer/ailayer_dense_default.h" +#include "basic/default/ailayer/ailayer_input_default.h" +#include "basic/default/ailayer/ailayer_relu_default.h" +#include "basic/default/ailayer/ailayer_leaky_relu_default.h" +#include "basic/default/ailayer/ailayer_elu_default.h" +#include "basic/default/ailayer/ailayer_sigmoid_default.h" +#include "basic/default/ailayer/ailayer_tanh_default.h" +#include "basic/default/ailayer/ailayer_softmax_default.h" +#include "basic/default/ailayer/ailayer_softsign_default.h" + +// Include the losses in default implementation +#include "basic/default/ailoss/ailoss_mse_default.h" +#include "basic/default/ailoss/ailoss_crossentropy_default.h" + +// Include the optimizers in default implementation +#include "basic/default/aiopti/aiopti_sgd_default.h" +#include "basic/default/aiopti/aiopti_adam_default.h" + +// ---------------------------- CMSIS implementations ----------------------- +// ATTENTION! +// If you want to use CMSIS for ARM controllers, you need to uncomment the define of AIFES_WITH_CMSIS + +//#define AIFES_WITH_CMSIS + +#ifdef AIFES_WITH_CMSIS + +// Include the math in cmsis implementation +#include "basic/cmsis/aimath/aimath_f32_cmsis.h" + +// Include the layers in cmsis implementation +#include "basic/cmsis/ailayer/ailayer_dense_cmsis.h" + +#endif /* AIFES_USE_CMSIS */ + +// ---------------------------- Algorithmic ----------------------- + +// Include the algorithmic +#include "basic/base/aialgo/aialgo_sequential_inference.h" +#include "basic/base/aialgo/aialgo_sequential_training.h" + +#ifdef __cplusplus +} // End extern "C" +#endif diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aialgo/aialgo_sequential_inference.c b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aialgo/aialgo_sequential_inference.c new file mode 100644 index 0000000..b11b867 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aialgo/aialgo_sequential_inference.c @@ -0,0 +1,224 @@ +/** + * \file basic/base/aialgo/aialgo_sequential_inference.c + * \version 2.0alpha + * \date 20.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/aialgo/aialgo_sequential_inference.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +#include +#include +#include + +uint32_t aialgo_sizeof_inference_memory(aimodel_t *model) +{ + uint16_t i; + uint32_t memory = 0, max_memory = 0; + ailayer_t *layer_ptr = model->input_layer; + + // Calculate result shapes and max amount of memory + for(i = 0; i < model->layer_count; i++) + { + layer_ptr->calc_result_shape(layer_ptr); + memory = aimath_sizeof_tensor_data(&(layer_ptr->result)); + if(memory > max_memory) max_memory = memory; + + layer_ptr = layer_ptr->output_layer; + } + + return 2 * max_memory; // Input and output buffer +} + +uint32_t aialgo_sizeof_parameter_memory(aimodel_t *model) +{ + int i; + ailayer_t *layer_ptr = model->input_layer; + uint32_t memory = 0; + + for(i = 0; i < model->layer_count; i++) + { + layer_ptr->calc_result_shape(layer_ptr); + + // Memory for the quantization parameter of the intermediate results + memory += layer_ptr->result.dtype->tensor_params_size; + + // Memory for trainable parameter + if(layer_ptr->sizeof_paramem != 0) + { + memory += layer_ptr->sizeof_paramem(layer_ptr); + } + + layer_ptr = layer_ptr->output_layer; + } + return memory; +} + +void aialgo_distribute_parameter_memory(aimodel_t *model, void *memory_ptr, uint32_t memory_size) +{ + int i; + ailayer_t *layer_ptr = model->input_layer; + uint32_t address_counter = 0; + + for(i = 0; i < model->layer_count; i++) + { + layer_ptr->calc_result_shape(layer_ptr); + + // Memory for the quantization parameter of the intermediate results + if(layer_ptr->result.dtype->tensor_params_size != 0){ + layer_ptr->result.tensor_params = memory_ptr + address_counter; + address_counter += layer_ptr->result.dtype->tensor_params_size; + } + + // Memory for trainable parameter + if(layer_ptr->sizeof_paramem != 0) + { + layer_ptr->set_paramem(layer_ptr, memory_ptr + address_counter); + address_counter += layer_ptr->sizeof_paramem(layer_ptr); + } + + layer_ptr = layer_ptr->output_layer; + } + return; +} + + +uint8_t aialgo_schedule_inference_memory(aimodel_t *model, void *memory_ptr, uint32_t memory_size) +{ + uint16_t i; + ailayer_t *layer_ptr = model->input_layer; + + // Init result tensor with shape and memory + for(i = 0; i < model->layer_count; i++) + { + layer_ptr->calc_result_shape(layer_ptr); + layer_ptr->result.data = memory_ptr + (i % 2) * memory_size / 2; + + layer_ptr = layer_ptr->output_layer; + } + + return 0; +} + +aitensor_t *aialgo_forward_model(aimodel_t *model, aitensor_t *input_data) +{ + uint16_t i; + ailayer_t *layer_ptr = model->input_layer; + + model->input_layer->result.data = input_data->data; + model->input_layer->result.tensor_params = input_data->tensor_params; + for(i = 0; i < model->layer_count; i++) + { + layer_ptr->forward(layer_ptr); + + // Print intermediate results + //print_aitensor(&layer_ptr->result); + + layer_ptr = layer_ptr->output_layer; + } + + return &(model->output_layer->result); +} + +aitensor_t *aialgo_inference_model(aimodel_t *model, aitensor_t *input_data, aitensor_t *output_data) +{ + uint32_t i, j; + + uint16_t input_batch_shape[input_data->dim]; + aitensor_t input_batch = { + .dtype = input_data->dtype, + .shape = input_batch_shape, + .dim = input_data->dim, + .tensor_params = input_data->tensor_params + }; + aitensor_t *output_batch; + + uint32_t input_multiplier = 1; + for(i = input_data->dim - 1; i > 0; i--) + { + input_multiplier *= input_data->shape[i]; + input_batch_shape[i] = input_data->shape[i]; + } + input_batch_shape[0] = 1; + + uint32_t output_multiplier = 1; + for(i = output_data->dim - 1; i > 0; i--) + { + output_multiplier *= output_data->shape[i]; + } + + for(i = 0; i < input_data->shape[0]; i++) + { + input_batch.data = input_data->data + i * input_multiplier * input_data->dtype->size; + + output_batch = aialgo_forward_model(model, &input_batch); + + // ToDo: Copy tensor + for(j = 0; j < aimath_tensor_elements(output_batch); j++) + { + memcpy(output_data->data + i * output_multiplier * input_data->dtype->size, + output_batch->data, + aimath_tensor_elements(output_batch) * input_data->dtype->size); + } + } + memcpy(output_data->tensor_params, output_batch->tensor_params, output_batch->dtype->tensor_params_size); + return output_data; +} + +uint8_t aialgo_compile_model(aimodel_t *model) +{ + ailayer_t *layer_ptr = model->input_layer; + uint16_t layer_counter = 1; + const uint16_t MAX_LAYER_COUNT = 128; // May be an other value + + model->trainable_params_count = 0; + while(layer_ptr != model->output_layer && layer_counter < MAX_LAYER_COUNT) + { + layer_counter++; + model->trainable_params_count += layer_ptr->trainable_params_count; + + layer_ptr = layer_ptr->output_layer; + } + model->layer_count = layer_counter; + + return 0; +} + +void aialgo_print_model_structure(aimodel_t *model) +{ + int i; + ailayer_t *layer_ptr = model->input_layer; + + printf("Layers:\n"); + for(i = 0; i < model->layer_count; i++){ + if(layer_ptr->layer_type->print_specs != 0){ + printf("%4d: %s (%s) <", i + 1, layer_ptr->layer_type->name, layer_ptr->result.dtype->name); + layer_ptr->layer_type->print_specs(layer_ptr, printf); + printf(">\n"); + } else { + printf("%4d: No specification found for this layer.\n", i + 1); + } + layer_ptr = layer_ptr->output_layer; + } + return; +} + diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aialgo/aialgo_sequential_inference.h b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aialgo/aialgo_sequential_inference.h new file mode 100644 index 0000000..7b448b9 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aialgo/aialgo_sequential_inference.h @@ -0,0 +1,145 @@ +/** + * \file basic/base/aialgo/aialgo_sequential_inference.h + * \internal + * \date 20.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Functions required for inference of models + * \details The functions target memory allocation/scheduling, the calculation of the forward pass and quantization for model inference + */ + +#ifndef AIALGO_SEQUENTIAL_INFERENCE +#define AIALGO_SEQUENTIAL_INFERENCE + +#include "core/aifes_core.h" +#include "core/aifes_math.h" +#include "basic/base/aimath/aimath_basic.h" +#include "basic/default/ailayer/ailayer_dense_default.h" + +/** @brief Calculate the memory requirements for intermediate results of an inference + * + * This memory is mainly for the result buffers of the layers. + * + * Use aialgo_schedule_inference_memory() to set the memory to the model. + * + * @param *model The model + * @return Required memory size in bytes + */ +uint32_t aialgo_sizeof_inference_memory(aimodel_t *model); + +/** @brief Calculate the memory requirements for the trainable parameters (like weights, bias, ...) of the model + * + * Use aialgo_distribute_parameter_memory() to set the memory to the model. + * + * @param *model The model + * @return Required memory size in bytes + */ +uint32_t aialgo_sizeof_parameter_memory(aimodel_t *model); + +/** @brief Assign the memory for intermediate results of an inference to the model + * + * The required memory size can be calculated with aialgo_sizeof_inference_memory() + * + * @param *model The model + * @param *memory_ptr Pointer to the memory block + * @param memory_size Size of the memory block (for error checking) + * @return 0 if successful + */ +uint8_t aialgo_schedule_inference_memory(aimodel_t *model, void *memory_ptr, uint32_t memory_size); + +/** @brief Assign the memory for the trainable parameters (like weights, bias, ...) of the model + * + * Only use this function if the parameters are not pre-trained or manually configured. + * Afterwards an initialization of the memory (for example by initializing the weights) has to be performed. + * + * The required memory size can be calculated with aialgo_sizeof_parameter_memory() + * + * @param *model The model + * @param *memory_ptr Pointer to the memory block + * @param memory_size Size of the memory block (for error checking) + * @return 0 if successful + */ +void aialgo_distribute_parameter_memory(aimodel_t *model, void *memory_ptr, uint32_t memory_size); + +/** @brief Perform a forward pass on the model + * + * The result is stored in the result tensor of the output layer and a pointer to this is returned. + * This output result is stored in the inference memory and is only valid as long as the inference memory is valid. + * To get the output as a separate tensor, use aialgo_inference_model() instead. + * + * @param *model The model + * @param *input_data Input data tensor of the same shape as the input_layer shape + * @return Pointer to the output data of the forward pass (points to the result tensor of the output layer) + */ +aitensor_t *aialgo_forward_model(aimodel_t *model, aitensor_t *input_data); + +/** @brief Perform an inference on the model / Run the model + * + * Make shure to initialize the model (aialgo_compile_model()) and schedule the inference memory + * (for example with aialgo_schedule_inference_memory() or aialgo_schedule_training_memory()) before + * calling this function. + * + * Example: + * \code{.c} + * float input_data[] = {0.0f, 1.0f}; + * uint16_t input_shape[] = {1, 2} + * aitensor_t input_tensor = { + * .dtype = aif32, + * .dim = 2, + * .shape = input_shape, + * .data = input_data + * }; + * + * float output_data[1]; + * uint16_t output_shape[] = {1, 1} + * aitensor_t output_tensor = { + * .dtype = aif32, + * .dim = 2, + * .shape = output_shape, + * .data = output_data + * }; + * + * aialgo_inference_model(&model, &input_tensor, &output_tensor); + * + * // The results are now in the output_tensor + * \endcode + * + * @param *model The model + * @param *input_data Input data tensor of the same shape as the input_layer shape + * @param *output_data Empty tensor for the results of the inference with the size of your outputs + * @return Pointer to the output_data tensor with the results + */ +aitensor_t *aialgo_inference_model(aimodel_t *model, aitensor_t *input_data, aitensor_t *output_data); + +/** @brief Initialize the model structure +* +* Counts the number of layers and trainable parameters in a model as preparation for inference or training. +* +* @param *model The model +* @return 0 if successful +*/ +uint8_t aialgo_compile_model(aimodel_t *model); + +/** @brief Print the layer structure of the model with the configured parameters +* +* @param *model The model +*/ +void aialgo_print_model_structure(aimodel_t *model); + +#endif // AIALGO_SEQUENTIAL_INFERENCE diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aialgo/aialgo_sequential_training.c b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aialgo/aialgo_sequential_training.c new file mode 100644 index 0000000..cae7d3f --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aialgo/aialgo_sequential_training.c @@ -0,0 +1,299 @@ +/** + * \file basic/base/aialgo/aialgo_sequential_training.c + * \version 2.0alpha + * \date 20.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/aialgo/aialgo_sequential_training.h" +#include "basic/base/aialgo/aialgo_sequential_inference.h" + +// ToDo: Remove dependency +#include "basic/default/aimath/aimath_f32_default.h" + +uint32_t aialgo_sizeof_training_memory(aimodel_t *model, aiopti_t *optimizer) +{ + uint16_t i, j; + ailayer_t *layer_ptr = model->input_layer; + uint32_t memory = 0; + + for(i = 0; i < model->layer_count; i++) + { + // Result memory + layer_ptr->calc_result_shape(layer_ptr); + memory += aimath_sizeof_tensor_data(&(layer_ptr->result)); + + // Memory for the qantization parameter of the deltas + if(layer_ptr->output_layer->deltas.dtype != 0){ + memory += layer_ptr->output_layer->deltas.dtype->tensor_params_size; + } + + // Trainingmemory e.g. for gradients + if(layer_ptr->sizeof_trainmem != 0) + { + memory += layer_ptr->sizeof_trainmem(layer_ptr); + } + + // optimization memory (e.g. first or second momentum) + if(optimizer->sizeof_optimem != 0){ + for(j = 0; j < layer_ptr->trainable_params_count; j++){ + memory += optimizer->sizeof_optimem(optimizer, layer_ptr->trainable_params[j]); + } + } + + layer_ptr = layer_ptr->output_layer; + } + return memory; +} + +uint8_t aialgo_schedule_training_memory(aimodel_t *model, aiopti_t *optimizer, void *memory_ptr, uint32_t memory_size) +{ + uint16_t i, j; + uint32_t address_counter = 0; + ailayer_t *layer_ptr = model->input_layer; + + for(i = 0; i < model->layer_count; i++) + { + // Result memory = deltas memory + layer_ptr->calc_result_shape(layer_ptr); + layer_ptr->result.data = memory_ptr + address_counter; + + layer_ptr->output_layer->deltas.dtype = layer_ptr->result.dtype; + layer_ptr->output_layer->deltas.dim = layer_ptr->result.dim; + layer_ptr->output_layer->deltas.shape = layer_ptr->result.shape; + layer_ptr->output_layer->deltas.data = memory_ptr + address_counter; + address_counter += aimath_sizeof_tensor_data(&(layer_ptr->result)); + + // Memory for the qantization parameter of the deltas + if(layer_ptr->output_layer->deltas.dtype != 0){ + layer_ptr->output_layer->deltas.tensor_params = memory_ptr + address_counter; + address_counter += layer_ptr->output_layer->deltas.dtype->tensor_params_size; + } + + // Training memory e.g. for gradients + if(layer_ptr->sizeof_trainmem != 0) + { + layer_ptr->set_trainmem(layer_ptr, memory_ptr + address_counter); + address_counter += layer_ptr->sizeof_trainmem(layer_ptr); + } + + // optimization memory (e.g. first or second momentum) + if(optimizer->sizeof_optimem != 0){ + for(j = 0; j < layer_ptr->trainable_params_count; j++){ + layer_ptr->optimem[j] = memory_ptr + address_counter; + address_counter += optimizer->sizeof_optimem(optimizer, layer_ptr->trainable_params[j]); + } + } + + layer_ptr = layer_ptr->output_layer; + } + + return 0; +} + +void aialgo_init_model_for_training(aimodel_t *model, aiopti_t *optimizer) +{ + uint16_t i, j; + ailayer_t *layer_ptr = model->input_layer; + + for(i = 0; i < model->layer_count; i++) + { + // Init the optimization memory (e.g. setting the momentums to zero) + for(j = 0; j < layer_ptr->trainable_params_count; j++){ + if(optimizer->init_optimem != 0){ + optimizer->init_optimem(optimizer, layer_ptr->trainable_params[j], layer_ptr->gradients[j], layer_ptr->optimem[j]); + } + } + layer_ptr = layer_ptr->output_layer; + } + return; +} + +void aialgo_backward_model(aimodel_t *model, aitensor_t *target_data) +{ + uint16_t i; + ailayer_t *layer_ptr = model->output_layer; + + model->loss->calc_delta(model->loss, target_data); + for(i = 0; i < model->layer_count; i++) + { +#ifdef DEBUG_CHECKS + if(layer_ptr->backward == 0){ + printf("\nError: No backward function implementation in layer %d\n", i); + return; + } +#endif + layer_ptr->backward(layer_ptr); + layer_ptr = layer_ptr->input_layer; + } + return; +} + +void aialgo_train_model(aimodel_t *model, aitensor_t *input_tensor, aitensor_t *target_tensor, aiopti_t *optimizer, uint32_t batch_size) +{ + uint32_t i; + + aitensor_t input_batch; + uint16_t input_batch_shape[input_tensor->dim]; + input_batch.dtype = input_tensor->dtype; + input_batch.dim = input_tensor->dim; + input_batch.shape = input_batch_shape; + input_batch.tensor_params = input_tensor->tensor_params; + aitensor_t target_batch; + uint16_t target_batch_shape[target_tensor->dim]; + target_batch.dtype = target_tensor->dtype; + target_batch.dim = target_tensor->dim; + target_batch.shape = target_batch_shape; + target_batch.tensor_params = target_tensor->tensor_params; + + uint32_t input_multiplier = 1; + for(i = input_tensor->dim - 1; i > 0; i--) + { + input_multiplier *= input_tensor->shape[i]; + input_batch_shape[i] = input_tensor->shape[i]; + } + input_multiplier *= input_tensor->dtype->size; + input_batch_shape[0] = 1; + uint32_t target_multiplier = 1; + for(i = target_tensor->dim - 1; i > 0; i--) + { + target_multiplier *= target_tensor->shape[i]; + target_batch_shape[i] = target_tensor->shape[i]; + } + target_multiplier *= target_tensor->dtype->size; + target_batch_shape[0] = 1; + + uint32_t batch_count = (uint32_t) (input_tensor->shape[0] / batch_size); + uint32_t batch; + for(batch = 0; batch < batch_count; batch++) + { + aialgo_zero_gradients_model(model, optimizer); + for(i = 0; i < batch_size; i++) + { + input_batch.data = input_tensor->data + batch * input_multiplier * batch_size + i * input_multiplier; + target_batch.data = target_tensor->data + batch * target_multiplier * batch_size + i * target_multiplier; + + aialgo_forward_model(model, &input_batch); + aialgo_backward_model(model, &target_batch); + } + aialgo_update_params_model(model, optimizer); + } + return; +} + +void aialgo_calc_loss_model_f32(aimodel_t *model, aitensor_t *input_tensor, aitensor_t *target_tensor, float *result) +{ + uint32_t i; + float loss; + + aitensor_t input_batch; + uint16_t input_batch_shape[input_tensor->dim]; + input_batch.dtype = input_tensor->dtype; + input_batch.dim = input_tensor->dim; + input_batch.shape = input_batch_shape; + input_batch.tensor_params = input_tensor->tensor_params; + aitensor_t target_batch; + uint16_t target_batch_shape[target_tensor->dim]; + target_batch.dtype = target_tensor->dtype; + target_batch.dim = target_tensor->dim; + target_batch.shape = target_batch_shape; + target_batch.tensor_params = target_tensor->tensor_params; + + uint32_t input_multiplier = 1; + for(i = input_tensor->dim - 1; i > 0; i--) + { + input_multiplier *= input_tensor->shape[i]; + input_batch_shape[i] = input_tensor->shape[i]; + } + input_multiplier *= input_tensor->dtype->size; + input_batch_shape[0] = 1; + uint32_t target_multiplier = 1; + for(i = target_tensor->dim - 1; i > 0; i--) + { + target_multiplier *= target_tensor->shape[i]; + target_batch_shape[i] = target_tensor->shape[i]; + } + target_multiplier *= target_tensor->dtype->size; + target_batch_shape[0] = 1; + + *result = 0; + for(i = 0; i < input_tensor->shape[0]; i++) + { + input_batch.data = input_tensor->data + i * input_multiplier; + target_batch.data = target_tensor->data + i * target_multiplier; + + aialgo_forward_model(model, &input_batch); + model->loss->calc_loss(model->loss, &target_batch, &loss); + *result += loss; + } + return; +} + +void aialgo_zero_gradients_model(aimodel_t *model, aiopti_t *optimizer) +{ + uint16_t i, j; + ailayer_t *layer_ptr = model->input_layer; + + for(i = 0; i < model->layer_count; i++) + { + for(j = 0; j < layer_ptr->trainable_params_count; j++){ + optimizer->zero_gradients(optimizer, layer_ptr->gradients[j]); + } + layer_ptr = layer_ptr->output_layer; + } + return; +} + +void aialgo_update_params_model(aimodel_t *model, aiopti_t *optimizer) +{ + uint16_t i, j; + ailayer_t *layer_ptr = model->input_layer; + + if(optimizer->begin_step != 0){ + optimizer->begin_step(optimizer); + } + for(i = 0; i < model->layer_count; i++) + { + for(j = 0; j < layer_ptr->trainable_params_count; j++){ + optimizer->update_params(optimizer, layer_ptr->trainable_params[j], layer_ptr->gradients[j], layer_ptr->optimem[j]); + } + layer_ptr = layer_ptr->output_layer; + } + if(optimizer->end_step != 0){ + optimizer->end_step(optimizer); + } + return; +} + +void aialgo_print_loss_specs(ailoss_t *loss) +{ + printf("%s (%s) <", loss->loss_type->name, loss->connection_layer.deltas.dtype->name); + loss->loss_type->print_specs(loss, printf); + printf(">"); + return; +} + +void aialgo_print_optimizer_specs(aiopti_t *opti) +{ + printf("%s (%s) <", opti->optimizer_type->name, opti->dtype->name); + opti->optimizer_type->print_specs(opti, printf); + printf(">"); + return; +} diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aialgo/aialgo_sequential_training.h b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aialgo/aialgo_sequential_training.h new file mode 100644 index 0000000..5ba2aa4 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aialgo/aialgo_sequential_training.h @@ -0,0 +1,148 @@ +/** + * \file basic/base/aialgo/aialgo_sequential_training.h + * \internal + * \date 20.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Functions required for the training of models + * \details The functions target memory allocation/scheduling and the backpropagation for model training + */ + +#ifndef AIALGO_SEQUENTIAL_TRAINING +#define AIALGO_SEQUENTIAL_TRAINING + +#include "core/aifes_core.h" +#include "core/aifes_math.h" +#include "basic/base/aimath/aimath_basic.h" + +/** @brief Calculate the memory requirements for model training + * + * This memory is used for intermediate results, gradients and momentums. + * + * Use aialgo_schedule_training_memory() to set the memory to the model. + * + * @param *model The model + * @param *optimizer The optimizer that is used for training + * @return Required memory size in bytes + */ +uint32_t aialgo_sizeof_training_memory(aimodel_t *model, aiopti_t *optimizer); + +/** @brief Assign the memory for model training + * + * This memory is used for intermediate results, gradients and momentums. + * + * The required memory size can be calculated with aialgo_sizeof_training_memory(). + * + * @param *model The model + * @param *optimizer The optimizer that is used for training + * @param *memory_ptr Pointer to the memory block + * @param memory_size Size of the memory block (for error checking) + * @return 0 if successful + */ +uint8_t aialgo_schedule_training_memory(aimodel_t *model, aiopti_t *optimizer, void *memory_ptr, uint32_t memory_size); + +/** @brief Initialize the optimization memory of the model layers + * + * @param *model The model + * @param *optimizer The optimizer that is used for training + */ +void aialgo_init_model_for_training(aimodel_t *model, aiopti_t *optimizer); + +/** @brief Perform the backward pass + * + * @param *model The model + * @param *target_data The tensor containing the target data / labels + */ +void aialgo_backward_model(aimodel_t *model, aitensor_t *target_data); + +/** @brief Perform one training epoch on all data batches of the dataset using backpropagation + * + * Make shure to initialize the model (aialgo_compile_model()) and schedule the training memory + * (for example with aialgo_schedule_training_memory()) and initialize the training memory + * (aialgo_init_model_for_training()) before calling this function. + * + * Example: Training of an F32 model for multiple epochs + * \code{.c} + * int epochs = 100; + * int batch_size = 4; + * int print_interval = 10; + * + * float loss; + * for(i = 0; i < epochs; i++) + * { + * // One epoch of training. Iterates through the whole data once + * aialgo_train_model(&model, &input_tensor, &target_tensor, optimizer, batch_size); + + * // Calculate and print loss every print_interval epochs + * if(i % print_interval == 0) + * { + * aialgo_calc_loss_model_f32(&model, &input_tensor, &target_tensor, &loss); + * printf("Epoch %5d: loss: %f\n", i, loss); + * } + * } + * \endcode + * + * @param *model The model + * @param *input_tensor The tensor containing the input data + * @param *target_tensor The tensor containing the target data / labels + * @param *optimizer The optimizer that is used for training + * @param batch_size Size of a batch / Number of input vektors + */ +void aialgo_train_model(aimodel_t *model, aitensor_t *input_tensor, aitensor_t *target_tensor, aiopti_t *optimizer, uint32_t batch_size); + +/** @brief Calculate the loss in \link aimath_f32.h F32 \endlink data type + * + * @param *model The model + * @param *input_data Tensor containing the input data + * @param *target_data Tensor containing the target data / labels + * @param *result The calculated loss will be written here + */ +void aialgo_calc_loss_model_f32(aimodel_t *model, aitensor_t *input_data, aitensor_t *target_data, float *result); + +/** @brief Set the gradients to zero + * + * @param *model The model + * @param *optimizer The optimizer that is used for training + */ +void aialgo_zero_gradients_model(aimodel_t *model, aiopti_t *optimizer); + +/** @brief Perform the optimization step on the model parameters + * + * @param *model The model + * @param *optimizer The optimizer that is used for training + */ +void aialgo_update_params_model(aimodel_t *model, aiopti_t *optimizer); + +/** @brief Print the loss specs + * + * Prints information like type, data type and constants to the console. + * + * @param *loss The loss + */ +void aialgo_print_loss_specs(ailoss_t *loss); + +/** @brief Print the optimizer specs + * + * Prints information like type, data type and constants to the console. + * + * @param *opti The optimizer + */ +void aialgo_print_optimizer_specs(aiopti_t *opti); + +#endif // AIALGO_SEQUENTIAL_TRAINING diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_dense.c b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_dense.c new file mode 100644 index 0000000..d3306b4 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_dense.c @@ -0,0 +1,246 @@ +/** + * \file basic/base/ailayer/ailayer_dense.c + * \version 2.0alpha + * \date 20.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/ailayer/ailayer_dense.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_layertype_t ailayer_dense_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "Dense", + .print_specs = ailayer_dense_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_layertype_t *ailayer_dense_type = &ailayer_dense_type_s; + +ailayer_t *ailayer_dense(ailayer_dense_t *layer, ailayer_t *input_layer) +{ + layer->base.layer_type = ailayer_dense_type; + + layer->base.input_layer = input_layer; + input_layer->output_layer = &(layer->base); + + layer->base.layer_configuration = layer; + layer->base.result.dtype = layer->result_dtype; + layer->base.result.dim = 2; + layer->base.result.shape = layer->result_shape; + layer->base.result.shape[1] = layer->neurons; + + layer->base.deltas.dtype = layer->result_dtype; + layer->base.deltas.dim = 2; + layer->base.deltas.shape = input_layer->result.shape; + + layer->weights.dim = 2; + layer->weights.dtype = layer->weights_dtype; + layer->weights.shape = layer->weights_shape; + layer->weights.shape[0] = input_layer->result.shape[1]; + layer->weights.shape[1] = layer->neurons; + + layer->bias.dim = 2; + layer->bias.dtype = layer->bias_dtype; + layer->bias.shape = layer->bias_shape; + layer->bias.shape[0] = 1; + layer->bias.shape[1] = layer->neurons; + + layer->base.forward = ailayer_dense_forward; + layer->base.backward = ailayer_dense_backward; + + layer->base.calc_result_shape = ailayer_dense_calc_result_shape; + layer->base.sizeof_paramem = ailayer_dense_sizeof_paramem; + layer->base.set_paramem = ailayer_dense_set_paramem; + layer->base.sizeof_trainmem = ailayer_dense_sizeof_trainmem; + layer->base.set_trainmem = ailayer_dense_set_trainmem; + + layer->base.get_result_bound = 0; + + layer->base.trainable_params_count = 2; + layer->base.trainable_params = layer->trainable_params; + layer->base.gradients = layer->gradients; + layer->base.optimem = layer->optimem; + + layer->trainable_params[0] = &layer->weights; + layer->trainable_params[1] = &layer->bias; + + return &layer->base; +} + +void ailayer_dense_forward(ailayer_t *self) +{ + aitensor_t *input_tensor = &(self->input_layer->result); + aitensor_t *result_tensor = &(self->result); + ailayer_dense_t *layer = (ailayer_dense_t *)(self->layer_configuration); + aitensor_t *weight_tensor = &(layer->weights); + aitensor_t *bias_tensor = &(layer->bias); + + // z = x * W + b + layer->linear(input_tensor, weight_tensor, bias_tensor, result_tensor); + + return; +} + + +void ailayer_dense_backward(ailayer_t *self) +{ + aitensor_t *delta_in = &(self->deltas); + aitensor_t *delta_out = &(self->output_layer->deltas); + aitensor_t *x_in = &(self->input_layer->result); + ailayer_dense_t *layer = (ailayer_dense_t *)(self->layer_configuration); + aitensor_t *weights = &(layer->weights); + aitensor_t *d_weights = layer->gradients[0]; + aitensor_t *d_bias = layer->gradients[1]; + + int8_t temp_result_data[aimath_sizeof_tensor_data(d_weights)]; + aitensor_t temp_result = { + .dim = 2, + .shape = d_weights->shape, + .data = temp_result_data, + .dtype = d_weights->dtype, + .tensor_params = d_weights->tensor_params + }; + + aimath_transpose_vector(x_in); + // d_weights += x_in^T * delta_out + layer->mat_mul(x_in, delta_out, &temp_result); + layer->tensor_add(d_weights, &temp_result, d_weights); + aimath_transpose_vector(x_in); + // d_bias += delta_out + layer->tensor_add(d_bias, delta_out, d_bias); + + // Calculate delta for next layer. Do not before calculating gradients!!! May override x_in!!! + // d_in = w^T * d_out = (d_out^T * w)^T + aimath_transpose_vector(delta_out); + aimath_transpose_vector(delta_in); + layer->mat_mul(weights, delta_out, delta_in); + aimath_transpose_vector(delta_in); + aimath_transpose_vector(delta_out); + + return; +} + +void ailayer_dense_calc_result_shape(ailayer_t *self) +{ + ailayer_dense_t *layer = (ailayer_dense_t *)(self->layer_configuration); + + self->result.shape[0] = self->input_layer->result.shape[0]; + self->result.shape[1] = layer->neurons; + + return; +} + +uint32_t ailayer_dense_sizeof_paramem(const ailayer_t *self) +{ + uint32_t memory = 0; + ailayer_dense_t *layer = (ailayer_dense_t *)(self->layer_configuration); + + // Weights + memory += layer->weights_dtype->tensor_params_size; + memory += self->input_layer->result.shape[1] * layer->neurons * aimath_sizeof_dtype(layer->weights_dtype); // data + + // Bias + memory += layer->bias_dtype->tensor_params_size; + memory += layer->neurons * aimath_sizeof_dtype(layer->bias_dtype); // data + return memory; +} + +void ailayer_dense_set_paramem(ailayer_t *self, void *memory_ptr) +{ + uint32_t address_counter = 0; + ailayer_dense_t *layer = (ailayer_dense_t *) (self->layer_configuration); + + layer->weights.tensor_params = memory_ptr + address_counter; + address_counter += layer->weights_dtype->tensor_params_size; + layer->weights.dim = 2; + layer->weights.dtype = layer->weights_dtype; + layer->weights.shape = layer->weights_shape; + layer->weights.shape[0] = self->input_layer->result.shape[1]; + layer->weights.shape[1] = layer->neurons; + layer->weights.data = memory_ptr + address_counter; + address_counter += aimath_sizeof_tensor_data(&(layer->weights)); + + layer->bias.tensor_params = memory_ptr + address_counter; + address_counter += layer->bias_dtype->tensor_params_size; + layer->bias.dim = 2; + layer->bias.dtype = layer->bias_dtype; + layer->bias.shape = layer->bias_shape; + layer->bias.shape[0] = 1; + layer->bias.shape[1] = layer->neurons; + layer->bias.data = memory_ptr + address_counter; + //address_counter += aimath_sizeof_tensor_data(&(configuration->bias)); + + layer->trainable_params[0] = &(layer->weights); + layer->trainable_params[1] = &(layer->bias); + + return; +} + +uint32_t ailayer_dense_sizeof_trainmem(const ailayer_t *self) +{ + uint32_t memory = 0; + ailayer_dense_t *layer = (ailayer_dense_t *)(self->layer_configuration); + + memory += aimath_sizeof_tensor(&layer->weights); + memory += aimath_sizeof_tensor(&layer->bias); + return memory; +} + +void ailayer_dense_set_trainmem(ailayer_t *self, void *memory_ptr) +{ + uint32_t address_counter = 0; + ailayer_dense_t *layer = (ailayer_dense_t *) (self->layer_configuration); + + // Weights gradients in gradients[0] + self->gradients[0] = memory_ptr; + address_counter += sizeof(aitensor_t); + self->gradients[0]->data = memory_ptr + address_counter; + self->gradients[0]->dtype = layer->weights.dtype; + self->gradients[0]->dim = 2; + self->gradients[0]->shape = layer->weights.shape; + address_counter += aimath_sizeof_tensor_data(layer->gradients[0]); + self->gradients[0]->tensor_params = memory_ptr + address_counter; + address_counter += aimath_sizeof_tensor_params(layer->gradients[0]); + + // Bias gradients in gradients[1] + self->gradients[1] = memory_ptr + address_counter; + address_counter += sizeof(aitensor_t); + self->gradients[1]->data = memory_ptr + address_counter; + self->gradients[1]->dtype = layer->bias.dtype; + self->gradients[1]->dim = 2; + self->gradients[1]->shape = layer->bias.shape; + address_counter += aimath_sizeof_tensor_data(layer->gradients[1]); + self->gradients[1]->tensor_params = memory_ptr + address_counter; + address_counter += aimath_sizeof_tensor_params(layer->gradients[1]); + + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void ailayer_dense_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)) +{ + ailayer_dense_t *layer = (ailayer_dense_t *)(self->layer_configuration); + + print("neurons: %ld", (long unsigned int) layer->neurons); +} +#endif diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_dense.h b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_dense.h new file mode 100644 index 0000000..66cbd2d --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_dense.h @@ -0,0 +1,296 @@ +/** + * \file basic/base/ailayer/ailayer_dense.h + * \internal + * \date 20.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link ailayer layer \endlink implementation of the Dense layer + * + * This is an "abstract" data-type independent implementation. To use the layer use one of the provided + * implementations for a specific hardware and data-type (for example from ailayer_dense_default.h) or set + * the required math functions on your own. + * + * \image html ailayer_dense_schematic.png width=200px + * + * The Dense layer (or fully connected layer) is the core layer of a FNN and calculates the weighted sums of its inputs + * @f[ + * y = x \cdot W + b + * @f] + * with the number of inputs \f$ K \f$, the number of neurons / outputs \f$ M \f$, + * the input vektor \f$ x \in \mathbb{R}^{1 \times K} \f$, the output vektor \f$ y \in \mathbb{R}^{1 \times M} \f$, + * the weights matrix \f$ W \in \mathbb{R}^{K \times M} \f$ and the bias \f$ b \in \mathbb{R}^{1 \times M} \f$. + * + * To increase the computational speed it is also possible to process a whole data batch at ones: + * @f[ + * Y = X \cdot W \oplus b + * @f] + * with the number of inputs \f$ K \f$, the number of neurons / outputs \f$ M \f$, the batch size \f$ N \f$, + * the input matrix \f$ X \in \mathbb{R}^{N \times K} \f$, the output vektor \f$ Y \in \mathbb{R}^{N \times M} \f$, + * the weights matrix \f$ W \in \mathbb{R}^{K \times M} \f$ and the bias \f$ b \in \mathbb{R}^{1 \times M} \f$. + * In this case the \f$ \oplus \f$ means a broadcasted addition of the bias to the rows of the result of \f$ X \cdot W \f$: + * @f[ + * X \cdot W \oplus b \rightarrow X \cdot W + \left( \begin{array}{c} 1 \\ \vdots \\ 1 \\ \end{array}\right) \cdot b + * @f] + * Keep in mind that also the memory size for intermediate results increases with the batch size. + * + * The results of the forward pass of this layer are written to the result tensor of the base ailayer_t struct. + */ + +#ifndef AILAYER_DENSE +#define AILAYER_DENSE + +#include "core/aifes_core.h" +#include "basic/default/aimath/aimath_f32_default.h" + +#define DENSE_WEIGHTS_SIZE(INPUTS, OUTPUTS) ((INPUTS) * (OUTPUTS)) +#define DENSE_BIAS_SIZE(OUTPUTS) (OUTPUTS) + +#define DENSE_WEIGHTS_SHAPE(INPUTS, OUTPUTS) {INPUTS, OUTPUTS} +#define DENSE_BIAS_SHAPE(OUTPUTS) {1, OUTPUTS} + +typedef struct ailayer_dense ailayer_dense_t; + +/** @brief General \link ailayer_dense.h Dense layer \endlink structure +* +*/ +struct ailayer_dense { + ailayer_t base; /**< Inherited field members from general ailayer struct. */ + const aimath_dtype_t *result_dtype; /**< Data type of the inference result values. */ + + /** @name Layer configuration + * @brief Required configuration parameters for the layer + * + * These fields have to be configured by the user before calling the initializer function. + */ + ///@{ + uint32_t neurons; /**< Layer neurons count (number of outputs). */ + ///@} + + /** @name Trainable parameters + * @brief Data fields for the trainable parameters (weights, bias) of the layer + */ + ///@{ + aitensor_t weights; /**< Tensor containing the layer weights. */ + aitensor_t bias; /**< Tensor containing the layer bias weights. */ + + const aimath_dtype_t *weights_dtype; /**< Data type of the weights. */ + const aimath_dtype_t *bias_dtype; /**< Data type of the bias weights. */ + + uint16_t weights_shape[2]; /**< Weights tensor shape (n x m matrix). */ + uint16_t bias_shape[2]; /**< Bias weights tensor shape (n x m matrix). */ + + aitensor_t *trainable_params[2]; /**< Pointer to the weights and biases (which are the trainable parameters). */ + aitensor_t *gradients[2]; /**< Gradients structure for the back propagation algorithm. */ + void *optimem[2]; /**< Memory field used by the trainings optimizer. */ + ///@} + + /** @name Math functions + * @brief Required data type specific math functions + */ + ///@{ + + /** @brief Required math function: Linear transformation + * + * Requires a math function that performs a linear transformation:\n + * @f[ + * result = a \cdot b \oplus c = a \cdot b + \left( \begin{array}{c} 1 \\ \vdots \\ 1 \\ \end{array}\right) \cdot c + * @f] + * + * @param a Matrix with dimension \f$ N \times K \f$ (input) + * @param b Matrix with dimension \f$ K \times M \f$ (input) + * @param c Laying vektor with dimension \f$ 1 \times M \f$ (input) + * @param result Matrix with dimension \f$ N \times M \f$ (output) + */ + void (*linear)(const aitensor_t *a, const aitensor_t *b, const aitensor_t *c, aitensor_t *result); + + /** @brief Required math function: Matrix multiplication + * + * Requires a math function that performs a matrix multiplication on two 2D tensors:\n + * @f[ + * result = a \cdot b + * @f] + * + * @param a Matrix with dimension \f$ N \times K \f$ (input) + * @param b Matrix with dimension \f$ K \times M \f$ (input) + * @param result Matrix with dimension \f$ N \times M \f$ (output) + */ + void (*mat_mul)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + /** @brief Required math function: Element wise tensor addition + * + * Requires a math function that adds two tensors element wise:\n + * @f[ + * result = a + b + * @f] + */ + void (*tensor_add)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + ///@} + + uint16_t result_shape[2]; /**< Inference result tensor (ailayer.result) shape. */ +}; + +/** @brief Dense layer type + * + * Defines the type of the layer (for example for type checks and debug prints). + * See aicore_layertype for more information about the layer type. + */ +extern const aicore_layertype_t *ailayer_dense_type; + +/** @brief Initialize and connect the given Dense layer + * + * This function represents the "constructor" of the abstract Dense layer. It initializes the layer structure + * and connects it to the previous layer.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example ailayer_dense_f32_default()). + * + * @param *layer The layer to initialize. + * @param *input_layer The previous layer that provides the inputs to the layer. + * @return Pointer to the (successfully) initialized general layer structure (ailayer_dense.base) + */ +ailayer_t *ailayer_dense(ailayer_dense_t *layer, ailayer_t *input_layer); + +/** @brief Calculate the forward pass for given Dense layer + * + * *Implementation of ailayer.forward.* + * + * It uses the result tensor of the previous layer as input and writes the result of the forward pass + * to the result tensor (ailayer.result) of the given layer. + * + * Calculation of the forward pass result: + * @f[ + * x_{out} \leftarrow x_{in} \cdot w \oplus b + * @f] + * + * \f$ w \f$: Weights matrix\n + * \f$ b \f$: Bias vektor\n + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ x_{out} \f$: Result of the forward pass of this layer\n\n + * + * Used math functions: + * * ailayer_dense.linear + * + * @param *self Layer to calculate the forward path for. + */ +void ailayer_dense_forward(ailayer_t *self); + +/** @brief Calculate the backward pass for the given Dense layer + * + * *Implementation of ailayer.backward.* + * + * It uses the deltas tensor of the next layer as input and writes the result of the backward pass + * to the deltas tensor (ailayer.deltas) of the given layer. + * + * Calculation of the gradients: + * @f[ + * \partial w \leftarrow \partial w + x_{in}^T \cdot \delta_{out} + * @f] + * @f[ + * \partial b \leftarrow \partial b + \left( \begin{array}{c} 1 & \cdots & 1 \\ \end{array}\right) \cdot \delta_{out} + * @f] + * + * Calculation of the errors for the previous layer: + * @f[ + * \delta_{in} \leftarrow w^T \cdot \delta_{out} = (\delta_{out}^T \cdot w)^T + * @f] + * + * \f$ w \f$: Weights matrix\n + * \f$ b \f$: Bias vektor\n + * \f$ \partial w \f$: Gradients matrix for the weights\n + * \f$ \partial b \f$: Gradients vektor for the bias\n + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ \delta_{in} \f$: Result of the backward pass of this layer\n + * \f$ \delta_{out} \f$: Result of the backward pass of the next layer\n\n + * + * Used math functions: + * * ailayer_dense.mat_mul + * * ailayer_dense.tensor_add + * + * @param *self Layer to calculate the backward path for. + */ +void ailayer_dense_backward(ailayer_t *self); + +/** @brief Calculate the shape of the result tensor (ailayer.result) + * + * *Implementation of ailayer.calc_result_shape.* + * + * Resulting shape is [count_inputs x count_neurons] + * + * @param *self Layer to calculate the resulting shape for. + */ +void ailayer_dense_calc_result_shape(ailayer_t *self); + + +/** @brief Calculate and return the parameter memory size needed for this layer + * + * *Implementation of ailayer.sizeof_paramem.* + * + * The parameter size is calculated for the \link ailayer_dense.weights weights \endlink and + * \link ailayer_dense.bias bias \endlink tensors, including the data and tensor_params fields. + * + * @param *self The layer to calculate the parameter memory size for + * @return Calculated parameter memory size in bytes. + */ +uint32_t ailayer_dense_sizeof_paramem(const ailayer_t *self); + +/** @brief Distribute provided memory to the parameter pointers + * + * *Implementation of ailayer.set_paramem.* + * + * Distributes the given buffer to the parameter pointers and sets + * the tensor parameters for weights and bias holding structures.\n + * The required parameter size can be calculated with ailayer_dense_sizeof_paramem() + * + * @param *self The layer to set the memory fields for. + * @param *memory_ptr The memory that can be used for the parameters + */ +void ailayer_dense_set_paramem(ailayer_t *self, void *memory_ptr); + +/** @brief Calculate and return the memory size needed by this layer for training + * + * *Implementation of ailayer.sizeof_trainmem.* + * + * The memory size is calculated for the gradient tensors of weights and bias. + * + * @param *self The layer to calculate the gradient memory size for. + * @return Calculated gradient memory size in bytes. + */ +uint32_t ailayer_dense_sizeof_trainmem(const ailayer_t *self); + +/** @brief Distribute provided memory to the gradients pointers + * + * *Implementation of ailayer.set_trainmem.* + * + * The required memory size can be calculated with ailayer_dense_sizeof_trainmem(). + * + * @param *self The layer to set the memory fields for. + * @param *memory_ptr The memory that can be used for the gradients + */ +void ailayer_dense_set_trainmem(ailayer_t *self, void *memory_ptr); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the layer specification + * + * @param *self The layer to print the specification for + * @param *print Pointer to the print function to use + */ +void ailayer_dense_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // AILAYER_DENSE diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_elu.c b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_elu.c new file mode 100644 index 0000000..35aaa1f --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_elu.c @@ -0,0 +1,113 @@ +/** + * \file basic/base/ailayer/ailayer_elu.c + * \version 2.0alpha + * \date 16.03.2021 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/ailayer/ailayer_elu.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_layertype_t ailayer_elu_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "ELU", + .print_specs = ailayer_elu_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_layertype_t *ailayer_elu_type = &ailayer_elu_type_s; + +ailayer_t *ailayer_elu(ailayer_elu_t *layer, ailayer_t *input_layer) ////const void *beta1, +{ + layer->base.layer_type = ailayer_elu_type; + + layer->base.input_layer = input_layer; + input_layer->output_layer = &(layer->base); + + layer->base.layer_configuration = layer; + layer->base.result.dtype = layer->dtype; + layer->base.result.shape = input_layer->result.shape; + layer->base.result.dim = input_layer->result.dim; + + layer->base.deltas.dtype = layer->dtype; + layer->base.deltas.dim = 2; + layer->base.deltas.shape = layer->base.result.shape; + + layer->base.forward = ailayer_elu_forward; + layer->base.backward = ailayer_elu_backward; + + layer->base.calc_result_shape = ailayer_elu_calc_result_shape; + layer->base.sizeof_paramem = 0; + layer->base.set_paramem = 0; + layer->base.sizeof_trainmem = 0; + layer->base.set_trainmem = 0; + + layer->base.trainable_params_count = 0; + + return &(layer->base); +} + +void ailayer_elu_forward(ailayer_t *self) +{ + ailayer_elu_t *layer = (ailayer_elu_t *)(self->layer_configuration); + aitensor_t *x_in = &(self->input_layer->result); + aitensor_t *x_out = &(self->result); + + layer->elu(x_in, layer->alpha, x_out); + return; +} + + +void ailayer_elu_backward(ailayer_t *self) +{ + ailayer_elu_t *layer = (ailayer_elu_t *)(self->layer_configuration); + aitensor_t *delta_in = &(self->deltas); + aitensor_t *delta_out = &(self->output_layer->deltas); + aitensor_t *x_in = &(self->input_layer->result); + + // delta_in = delta_out .* elu'(x_in) + layer->d_elu(x_in, layer->alpha, delta_in); + layer->multiply(delta_in, delta_out, delta_in); + return; +} + +void ailayer_elu_calc_result_shape(ailayer_t *self) +{ + /* Unused: Shape is already defined (Pointer) + + self->result.shape[0] = self->input_layer->result.shape[0]; + self->result.shape[1] = self->input_layer->result.shape[1]; + */ + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void ailayer_elu_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)) +{ + ailayer_elu_t *self_casted = (ailayer_elu_t *) self->layer_configuration; + + print("alpha: "); + self_casted->dtype->print_aiscalar(self_casted->alpha, print); + return; +} +#endif + diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_elu.h b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_elu.h new file mode 100644 index 0000000..4a5bde3 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_elu.h @@ -0,0 +1,200 @@ +/** + * \file basic/base/ailayer/ailayer_elu.h + * \internal + * \date 15.03.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link ailayer layer \endlink implementation of the ELU activation layer + * + * This is an "abstract" data-type independent implementation. To use the layer use one of the provided + * implementations for a specific hardware and data-type (for example from ailayer_elu_default.h) or set + * the required math functions on your own. + * + * \image html ailayer_elu_schematic.png width=200px + * + * The ELU layer is used as an activation function layer right after a dense layer. It calculates + * @f[ + * y = \begin{cases} + \alpha (e^x - 1) & \text{if } x \leq 0\\ + x & \text{if } x > 0 + \end{cases} + * @f] + * for every element of the input tensor. + * + * The results of the forward pass of this layer are written to the result tensor of the base ailayer_t struct. + */ + +#ifndef ELU_LAYER +#define ELU_LAYER + +#include "core/aifes_core.h" + +typedef struct ailayer_elu ailayer_elu_t; + +/** @brief General \link ailayer_elu.h ELU layer \endlink struct +* +*/ +struct ailayer_elu { + ailayer_t base; /**< Inherited field members from general ailayer struct. */ + const aimath_dtype_t *dtype; /**< Data type of the input and inference result values. */ + + /** @name Layer configuration + * @brief Required configuration parameters for the layer + * + * These fields have to be configured by the user before calling the initializer function. + */ + ///@{ + void *alpha; /**< Parameter \f$ \alpha \f$ used to calculate ELU function for input values < 0. */ + ///@} + + /** @name Math functions + * @brief Required data type specific math functions + */ + ///@{ + + /** @brief Required math function: ELU + * + * Requires a math function that calculates the element wise ELU of a tensor:\n + * @f[ + * result_{i} = \begin{cases} + \alpha \cdot (e^{x_i} - 1) & \text{if } x_i < 0 \\ + x_i & \text{if } x_i \geq 0 + \end{cases} + * @f] + * + * @param x N-dimensional tensor (input) + * @param result N-dimensional tensor (output) + */ + void (*elu)(const aitensor_t *x, const void *alpha, aitensor_t *result); + + /** @brief Required math function: Derivative of ELU + * + * Requires a math function that calculates the element wise ELU derivative of a tensor:\n + * @f[ + * result_{i} = \begin{cases} + \alpha \cdot e^{x_i} & \text{if } x_i < 0\\ + 1 & \text{if } x_i \geq 0 + \end{cases} + * @f] + * + * @param x N-dimensional tensor (input) + * @param result N-dimensional tensor (output) + */ + void (*d_elu)(const aitensor_t *x, const void *alpha, aitensor_t *result); + + /** @brief Required math function: Element wise tensor multiplication + * + * Requires a math function that multiplies two tensors element wise:\n + * @f[ + * result = a \circ b + * @f] + */ + void (*multiply)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + ///@} +}; + +/** @brief ELU layer type + * + * Defines the type of the layer (for example for type checks and debug prints). + * See aicore_layertype for more information about the layer type. + */ +extern const aicore_layertype_t *ailayer_elu_type; + +/** @brief Initialize and connect the given ELU layer + * + * This function represents the "constructor" of the abstract ELU layer. It initializes the layer structure + * and connects it to the previous layer.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example ailayer_elu_f32_default()). + * + * @param *layer The layer to initialize. + * @param *input_layer The previous layer that provides the inputs to the layer. + * @return Pointer to the (successfully) initialized general layer structure (ailayer_elu.base) + */ +ailayer_t *ailayer_elu(ailayer_elu_t *layer, ailayer_t *input_layer); + +/** @brief Calculate the forward pass for given ELU layer + * + * *Implementation of ailayer.forward.* + * + * It uses the result tensor of the previous layer as input and writes the result of the forward pass + * to the result tensor (ailayer.result) of the given layer. + * + * Calculation of the forward pass result: + * @f[ + * x_{out} \leftarrow ELU(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ x_{out} \f$: Result of the forward pass of this layer\n\n + * + * Used math functions: + * * ailayer_elu.elu + * + * @param *self Layer to calculate the forward path for. + */ +void ailayer_elu_forward(ailayer_t *self); + +/** @brief Calculate the backward pass for the given ELU layer + * + * *Implementation of ailayer.backward.* + * + * It uses the deltas tensor of the next layer as input and writes the result of the backward pass + * to the deltas tensor (ailayer.deltas) of the given layer. + * + * Calculation of the errors for the previous layer: + * @f[ + * \delta_{in} \leftarrow \delta_{out} \circ ELU'(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ \delta_{in} \f$: Result of the backward pass of this layer\n + * \f$ \delta_{out} \f$: Result of the backward pass of the next layer\n\n + * + * Used math functions: + * * ailayer_elu.elu + * * ailayer_elu.d_elu + * * ailayer_elu.multiply + * + * @param *self Layer to calculate the backward path for. + */ +void ailayer_elu_backward(ailayer_t *self); + +/** @brief Calculate the shape of the result tensor + * + * *Implementation of ailayer.calc_result_shape.* + * + * As the result tensor shape is shared with the result tensor shape of the previous layer (no change in shape is needed), + * this function returns without doing anything. + * + * @param *self Layer to calculate the resulting shape for. + */ +void ailayer_elu_calc_result_shape(ailayer_t *self); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the layer specification + * + * @param *self The layer to print the specification for + * @param *print Pointer to the print function to use + */ +void ailayer_elu_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // ELU_LAYER diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_input.c b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_input.c new file mode 100644 index 0000000..7c8213e --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_input.c @@ -0,0 +1,99 @@ +/** + * \file basic/base/ailayer/ailayer_input.c + * \version 2.0alpha + * \date 20.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/ailayer/ailayer_input.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_layertype_t ailayer_input_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "Input", + .print_specs = ailayer_input_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_layertype_t *ailayer_input_type = &ailayer_input_type_s; + + +ailayer_t *ailayer_input(ailayer_input_t *layer) +{ + layer->base.layer_type = ailayer_input_type; + + layer->base.layer_configuration = layer; + layer->base.result.dtype = layer->dtype; + layer->base.result.shape = layer->input_shape; + layer->base.result.dim = layer->input_dim; + + layer->base.deltas.dtype = 0; + + layer->base.forward = ailayer_input_forward; + layer->base.backward = ailayer_input_backward; + + layer->base.calc_result_shape = ailayer_input_calc_result_shape; + layer->base.sizeof_paramem = 0; + layer->base.set_paramem = 0; + layer->base.sizeof_trainmem = 0; + layer->base.set_trainmem = 0; + + layer->base.get_result_bound = 0; + + layer->base.trainable_params_count = 0; + + return &layer->base; +} + +void ailayer_input_forward(ailayer_t *self) +{ + return; +} + +void ailayer_input_backward(ailayer_t *self) +{ + return; +} + +void ailayer_input_calc_result_shape(ailayer_t *self) +{ + //ailayer_input_t *layer = (ailayer_input_t *)(self->layer_configuration); + + //self->result.shape[0] = configuration->input_shape[0]; + //self->result.shape[1] = configuration->input_shape[1]; + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void ailayer_input_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)) +{ + ailayer_input_t *self_casted = (ailayer_input_t *) self; + uint8_t i = 0; + + print("Dim: %d; Shape: [%d", self_casted->input_dim, self_casted->input_shape[i]); + for(i = 1; i < self_casted->input_dim; i++){ + print(", %d", self_casted->input_shape[i]); + } + print("]"); + return; +} +#endif diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_input.h b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_input.h new file mode 100644 index 0000000..66ed268 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_input.h @@ -0,0 +1,120 @@ +/** + * \file ailayer/ailayer_input.h + * \internal + * \date 20.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link ailayer layer \endlink implementation of the Input layer + * + * This is an "abstract" data-type independent implementation. To use the layer use one of the provided + * implementations for a specific hardware and data-type (for example from ailayer_input_default.h) or set + * the required math functions on your own. + * + * The Input layer is the start layer for every AIfES 2 model. + */ + +#ifndef INPUT_LAYER +#define INPUT_LAYER + +#include "core/aifes_core.h" + +typedef struct ailayer_input ailayer_input_t; + +/** @brief General \link ailayer_input.h Input layer \endlink structure +* +*/ +struct ailayer_input { + ailayer_t base; /**< Inherited field members from general ailayer struct. */ + const aimath_dtype_t *dtype; /**< Data type of the input and inference result values. */ + + /** @name Layer configuration + * @brief Required configuration parameters for the layer + * + * These fields have to be configured by the user before calling the initializer function. + */ + ///@{ + uint8_t input_dim; /**< Dimension of the input tensor. */ + uint16_t *input_shape; /**< Shape of the input tensor. */ + ///@} +}; + +/** @brief Input layer type + * + * Defines the type of the layer (for example for type checks and debug prints). + * See aicore_layertype for more information about the layer type. + */ +extern const aicore_layertype_t *ailayer_input_type; + +/** @brief Initialize the given Input layer + * + * This function represents the "constructor" of the abstract Input layer. It initializes the layer structure + * and connects it to the previous layer.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example ailayer_input_f32_default()). + * + * @param *layer The layer to initialize. + * @return Pointer to the (successfully) initialized general layer structure (ailayer_input.base). + */ +ailayer_t *ailayer_input(ailayer_input_t *layer); + +/** @brief Calculate the forward pass for given Input layer + * + * *Implementation of ailayer.forward.* + * + * The standard layer interface demands a forward pass function + * for the net scheduler, but theres no sense in 'calculating' + * an input layer, so this function returns without doing something. + * + * @param *self Layer to 'calculate' the forward pass for. + */ +void ailayer_input_forward(ailayer_t *self); + +/** @brief Calculate the backward pass for the given Input layer + * + * *Implementation of ailayer.backward.* + * + * The standard layer interface demands a backward pass function + * for the net scheduler, but theres no sense in 'calculating' + * an input layer, so this function returns without doing something. + * + * @param *self Layer to 'calculate' the backward pass for. + */ +void ailayer_input_backward(ailayer_t *self); + +/** @brief Calculate the shape of the result tensor + * + * *Implementation of ailayer.calc_result_shape* + * + * As the result shape is given by the configured ailayer_input.input_shape parameter, + * this function returns without doing anything. + * + * @param *self Layer to calculate the resulting shape for. + */ +void ailayer_input_calc_result_shape(ailayer_t *self); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the layer specification + * + * @param *self The layer to print the specification for + * @param *print Pointer to the print function to use + */ +void ailayer_input_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // INPUT_LAYER diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_leaky_relu.c b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_leaky_relu.c new file mode 100644 index 0000000..0f73b81 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_leaky_relu.c @@ -0,0 +1,112 @@ +/** + * \file basic/base/ailayer/ailayer_leaky_relu.c + * \version 2.0alpha + * \date 16.03.2021 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/ailayer/ailayer_leaky_relu.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_layertype_t ailayer_leaky_relu_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "Leaky ReLU", + .print_specs = ailayer_leaky_relu_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_layertype_t *ailayer_leaky_relu_type = &ailayer_leaky_relu_type_s; + +ailayer_t *ailayer_leaky_relu(ailayer_leaky_relu_t *layer, ailayer_t *input_layer) ////const void *beta1, +{ + layer->base.layer_type = ailayer_leaky_relu_type; + + layer->base.input_layer = input_layer; + input_layer->output_layer = &(layer->base); + layer->base.layer_configuration = layer; + layer->base.result.dtype = layer->dtype; + layer->base.result.shape = input_layer->result.shape; + layer->base.result.dim = input_layer->result.dim; + + layer->base.deltas.dtype = layer->dtype; + layer->base.deltas.dim = 2; + layer->base.deltas.shape = layer->base.result.shape; + + layer->base.forward = ailayer_leaky_relu_forward; + layer->base.backward = ailayer_leaky_relu_backward; + + layer->base.calc_result_shape = ailayer_leaky_relu_calc_result_shape; + layer->base.sizeof_paramem = 0; + layer->base.set_paramem = 0; + layer->base.sizeof_trainmem = 0; + layer->base.set_trainmem = 0; + + layer->base.trainable_params_count = 0; + + return &(layer->base); +} + +void ailayer_leaky_relu_forward(ailayer_t *self) +{ + ailayer_leaky_relu_t *layer = (ailayer_leaky_relu_t *)(self->layer_configuration); + aitensor_t *x_in = &(self->input_layer->result); + aitensor_t *x_out = &(self->result); + + layer->leaky_relu(x_in, layer->alpha, x_out); + return; +} + +void ailayer_leaky_relu_backward(ailayer_t *self) +{ + ailayer_leaky_relu_t *layer = (ailayer_leaky_relu_t *)(self->layer_configuration); + aitensor_t *delta_in = &(self->deltas); + aitensor_t *delta_out = &(self->output_layer->deltas); + aitensor_t *x_in = &(self->input_layer->result); + + // delta_in = delta_out .* relu'(x_in) + layer->d_leaky_relu(x_in, layer->alpha, delta_in); + layer->multiply(delta_in, delta_out, delta_in); + + return; +} + +void ailayer_leaky_relu_calc_result_shape(ailayer_t *self) +{ + /* Unused: Shape is already defined (Pointer) + + self->result.shape[0] = self->input_layer->result.shape[0]; + self->result.shape[1] = self->input_layer->result.shape[1]; + */ + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void ailayer_leaky_relu_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)) +{ + ailayer_leaky_relu_t *self_casted = (ailayer_leaky_relu_t *) self; + + print("alpha: "); + self_casted->dtype->print_aiscalar(self_casted->alpha, print); + return; +} +#endif + diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_leaky_relu.h b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_leaky_relu.h new file mode 100644 index 0000000..c012ea2 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_leaky_relu.h @@ -0,0 +1,200 @@ +/** + * \file basic/base/ailayer/ailayer_leaky_relu.h + * \internal + * \date 15.03.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link ailayer layer \endlink implementation of the Leaky ReLU activation layer + * + * This is an "abstract" data-type independent implementation. To use the layer use one of the provided + * implementations for a specific hardware and data-type (for example from ailayer_leaky_relu_default.h) or set + * the required math functions on your own. + * + * \image html ailayer_leaky_relu_schematic.png width=200px + * + * The Leaky ReLU layer is used as an activation function layer right after a dense layer. It calculates + * @f[ + * y = \begin{cases} + \alpha x & \text{if } x < 0\\ + x & \text{if } x \geq 0 + \end{cases} + * @f] + * for every element of the input tensor. + * + * The results of the forward pass of this layer are written to the result tensor of the base ailayer_t struct. + */ + +#ifndef LEAKY_RELU_LAYER +#define LEAKY_RELU_LAYER + +#include "core/aifes_core.h" + +typedef struct ailayer_leaky_relu ailayer_leaky_relu_t; + +/** @brief General \link ailayer_leaky_relu.h Leaky ReLU layer \endlink struct +* +*/ +struct ailayer_leaky_relu { + ailayer_t base; /**< Inherited field members from general ailayer struct. */ + const aimath_dtype_t *dtype; /**< Data type of the input and inference result values. */ + + /** @name Layer configuration + * @brief Required configuration parameters for the layer + * + * These fields have to be configured by the user before calling the initializer function. + */ + ///@{ + void *alpha; /**< Parameter \f$ \alpha \f$ used to calculate Leaky ReLU function for input values < 0. */ + ///@} + + /** @name Math functions + * @brief Required data type specific math functions + */ + ///@{ + + /** @brief Required math function: Leaky ReLU + * + * Requires a math function that calculates the element wise Leaky ReLU of a tensor:\n + * @f[ + * result_{i} = \begin{cases} + \alpha \cdot x_i & \text{if } x_i < 0 \\ + x_i & \text{if } x_i \geq 0 + \end{cases} + * @f] + * + * @param x N-dimensional tensor (input) + * @param result N-dimensional tensor (output) + */ + void (*leaky_relu)(const aitensor_t *x, const void *alpha, aitensor_t *result); + + /** @brief Required math function: Derivative of Leaky ReLU + * + * Requires a math function that calculates the element wise Leaky ReLU derivative of a tensor:\n + * @f[ + * result_{i} = \begin{cases} + \alpha & \text{if } x_i < 0\\ + 1 & \text{if } x_i \geq 0 + \end{cases} + * @f] + * + * @param x N-dimensional tensor (input) + * @param result N-dimensional tensor (output) + */ + void (*d_leaky_relu)(const aitensor_t *x, const void *alpha, aitensor_t *result); + + /** @brief Required math function: Element wise tensor multiplication + * + * Requires a math function that multiplies two tensors element wise:\n + * @f[ + * result = a \circ b + * @f] + */ + void (*multiply)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + ///@} +}; + +/** @brief Leaky ReLU layer type + * + * Defines the type of the layer (for example for type checks and debug prints). + * See aicore_layertype for more information about the layer type. + */ +extern const aicore_layertype_t *ailayer_leaky_relu_type; + +/** @brief Initialize and connect the given Leaky ReLU layer + * + * This function represents the "constructor" of the abstract Leaky ReLU layer. It initializes the layer structure + * and connects it to the previous layer.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example ailayer_leaky_relu_f32_default()). + * + * @param *layer The layer to initialize. + * @param *input_layer The previous layer that provides the inputs to the layer. + * @return Pointer to the (successfully) initialized general layer structure (ailayer_leaky_relu.base). + */ +ailayer_t *ailayer_leaky_relu(ailayer_leaky_relu_t *layer, ailayer_t *input_layer); + +/** @brief Calculate the forward pass for given Leaky ReLU layer + * + * *Implementation of ailayer.forward.* + * + * It uses the result tensor of the previous layer as input and writes the result of the forward pass + * to the result tensor (ailayer.result) of the given layer. + * + * Calculation of the forward pass result: + * @f[ + * x_{out} \leftarrow LeakyReLU(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ x_{out} \f$: Result of the forward pass of this layer\n\n + * + * Used math functions: + * * ailayer_leaky_relu.leaky_relu + * + * @param *self Layer to calculate the forward path for. + */ +void ailayer_leaky_relu_forward(ailayer_t *self); + +/** @brief Calculate the backward pass for the given Leaky ReLU layer + * + * *Implementation of ailayer.backward.* + * + * It uses the deltas tensor of the next layer as input and writes the result of the backward pass + * to the deltas tensor (ailayer.deltas) of the given layer. + * + * Calculation of the errors for the previous layer: + * @f[ + * \delta_{in} \leftarrow \delta_{out} \circ LeakyReLU'(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ \delta_{in} \f$: Result of the backward pass of this layer\n + * \f$ \delta_{out} \f$: Result of the backward pass of the next layer\n\n + * + * Used math functions: + * * ailayer_leaky_relu.leaky_relu + * * ailayer_leaky_relu.d_leaky_relu + * * ailayer_leaky_relu.multiply + * + * @param *self Layer to calculate the backward path for. + */ +void ailayer_leaky_relu_backward(ailayer_t *self); + +/** @brief Calculate the shape of the result tensor + * + * *Implementation of ailayer.calc_result_shape.* + * + * As the result tensor shape is shared with the result tensor shape of the previous layer (no change in shape is needed), + * this function returns without doing anything. + * + * @param *self Layer to calculate the resulting shape for. + */ +void ailayer_leaky_relu_calc_result_shape(ailayer_t *self); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the layer specification + * + * @param *self The layer to print the specification for + * @param *print Pointer to the print function to use + */ +void ailayer_leaky_relu_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // LEAKY_RELU_LAYER diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_relu.c b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_relu.c new file mode 100644 index 0000000..e252381 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_relu.c @@ -0,0 +1,110 @@ +/** + * \file basic/base/ailayer/ailayer_relu.c + * \version 2.0alpha + * \date 07.12.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/ailayer/ailayer_relu.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_layertype_t ailayer_relu_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "ReLU", + .print_specs = ailayer_relu_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_layertype_t *ailayer_relu_type = &ailayer_relu_type_s; + + +ailayer_t *ailayer_relu(ailayer_relu_t *layer, ailayer_t *input_layer) +{ + layer->base.layer_type = ailayer_relu_type; + + layer->base.input_layer = input_layer; + input_layer->output_layer = &(layer->base); + + layer->base.layer_configuration = layer; + layer->base.result.dtype = layer->dtype; + layer->base.result.shape = input_layer->result.shape; + layer->base.result.dim = input_layer->result.dim; + + layer->base.deltas.dtype = layer->dtype; + layer->base.deltas.dim = 2; + layer->base.deltas.shape = layer->base.result.shape; + + layer->base.forward = ailayer_relu_forward; + layer->base.backward = ailayer_relu_backward; + + layer->base.calc_result_shape = ailayer_relu_calc_result_shape; + layer->base.sizeof_paramem = 0; + layer->base.set_paramem = 0; + layer->base.sizeof_trainmem = 0; + layer->base.set_trainmem = 0; + + layer->base.trainable_params_count = 0; + + return &(layer->base); +} + +void ailayer_relu_forward(ailayer_t *self) +{ + ailayer_relu_t *layer = (ailayer_relu_t *)(self->layer_configuration); + aitensor_t *x_in = &(self->input_layer->result); + aitensor_t *x_out = &(self->result); + + layer->relu(x_in, x_out); + return; +} + + +void ailayer_relu_backward(ailayer_t *self) +{ + ailayer_relu_t *layer = (ailayer_relu_t *)(self->layer_configuration); + aitensor_t *delta_in = &(self->deltas); + aitensor_t *delta_out = &(self->output_layer->deltas); + aitensor_t *x_in = &(self->input_layer->result); + + // delta_in = delta_out .* relu'(x_in) + layer->d_relu(x_in, delta_in); + layer->multiply(delta_in, delta_out, delta_in); + return; +} + +void ailayer_relu_calc_result_shape(ailayer_t *self) +{ + /* Unused: Shape is already defined (Pointer) + + self->result.shape[0] = self->input_layer->result.shape[0]; + self->result.shape[1] = self->input_layer->result.shape[1]; + */ + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void ailayer_relu_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)) +{ + return; +} +#endif + diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_relu.h b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_relu.h new file mode 100644 index 0000000..469aeaf --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_relu.h @@ -0,0 +1,187 @@ + +/** + * \file basic/base/ailayer/ailayer_relu.h + * \internal + * \date 07.12.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link ailayer layer \endlink implementation of the ReLU activation layer + * + * This is an "abstract" data-type independent implementation. To use the layer use one of the provided + * implementations for a specific hardware and data-type (for example from ailayer_relu_default.h) or set + * the required math functions on your own. + * + * \image html ailayer_relu_schematic.png width=200px + * + * The ReLU layer is used as an activation function layer right after a dense layer. It calculates + * @f[ + * y = \begin{cases} + 0 & \text{if } x < 0\\ + x & \text{if } x \geq 0 + \end{cases} + * @f] + * for every element of the input tensor. + * + * The results of the forward pass of this layer are written to the result tensor of the base ailayer_t struct. + */ + +#ifndef RELU_LAYER +#define RELU_LAYER + +#include "core/aifes_core.h" + +typedef struct ailayer_relu ailayer_relu_t; + +/** @brief General \link ailayer_relu.h ReLU layer \endlink struct +* +*/ +struct ailayer_relu { + ailayer_t base; /**< Inherited field members from general ailayer struct. */ + const aimath_dtype_t *dtype; /**< Data type of the input and inference result values. */ + + /** @name Math functions + * @brief Required data type specific math functions + */ + ///@{ + + /** @brief Required math function: ReLU + * + * Requires a math function that calculates the element wise ReLU of a tensor:\n + * @f[ + * result_{i} = max(0, x_{i}) + * @f] + * + * @param x N-dimensional tensor (input) + * @param result N-dimensional tensor (output) + */ + void (*relu)(const aitensor_t *x, aitensor_t *result); + + /** @brief Required math function: Derivative of ReLU + * + * Requires a math function that calculates the element wise ReLU derivative of a tensor:\n + * @f[ + * result_{i} = \begin{cases} + 0 & \text{if } x_i < 0\\ + 1 & \text{if } x_i \geq 0 + \end{cases} + * @f] + * + * @param x N-dimensional tensor (input) + * @param result N-dimensional tensor (output) + */ + void (*d_relu)(const aitensor_t *x, aitensor_t *result); + + /** @brief Required math function: Element wise tensor multiplication + * + * Requires a math function that multiplies two tensors element wise:\n + * @f[ + * result = a \circ b + * @f] + */ + void (*multiply)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); +}; + +/** @brief ReLU layer type + * + * Defines the type of the layer (for example for type checks and debug prints). + * See aicore_layertype for more information about the layer type. + */ +extern const aicore_layertype_t *ailayer_relu_type; + +/** @brief Initialize and connect the given ReLU layer + * + * This function represents the "constructor" of the abstract ReLU layer. It initializes the layer structure + * and connects it to the previous layer.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example ailayer_relu_f32_default()). + * + * @param *layer The layer to initialize. + * @param *input_layer The previous layer that provides the inputs to the layer. + * @return Pointer to the (successfully) initialized general layer structure (ailayer_relu.base). + */ +ailayer_t *ailayer_relu(ailayer_relu_t *layer, ailayer_t *input_layer); + +/** @brief Calculate the forward pass for given ReLU layer + * + * *Implementation of ailayer.forward.* + * + * It uses the result tensor of the previous layer as input and writes the result of the forward pass + * to the result tensor (ailayer.result) of the given layer. + * + * Calculation of the forward pass result: + * @f[ + * x_{out} \leftarrow ReLU(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ x_{out} \f$: Result of the forward pass of this layer\n\n + * + * Used math functions: + * * ailayer_relu.relu + * + * @param *self Layer to calculate the forward path for. + */ +void ailayer_relu_forward(ailayer_t *self); + +/** @brief Calculate the backward pass for the given ReLU layer + * + * *Implementation of ailayer.backward.* + * + * It uses the deltas tensor of the next layer as input and writes the result of the backward pass + * to the deltas tensor (ailayer.deltas) of the given layer. + * + * Calculation of the errors for the previous layer: + * @f[ + * \delta_{in} \leftarrow \delta_{out} \circ ReLU'(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ \delta_{in} \f$: Result of the backward pass of this layer\n + * \f$ \delta_{out} \f$: Result of the backward pass of the next layer\n\n + * + * Used math functions: + * * ailayer_relu.relu + * * ailayer_relu.d_relu + * * ailayer_relu.multiply + * + * @param *self Layer to calculate the backward path for. + */ +void ailayer_relu_backward(ailayer_t *self); + +/** @brief Calculate the shape of the result tensor + * + * *Implementation of ailayer.calc_result_shape.* + * + * As the result tensor shape is shared with the result tensor shape of the previous layer (no change in shape is needed), + * this function returns without doing anything. + * + * @param *self Layer to calculate the resulting shape for. + */ +void ailayer_relu_calc_result_shape(ailayer_t *self); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the layer specification + * + * @param *self The layer to print the specification for + * @param *print Pointer to the print function to use + */ +void ailayer_relu_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // RELU_LAYER diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_sigmoid.c b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_sigmoid.c new file mode 100644 index 0000000..21b9252 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_sigmoid.c @@ -0,0 +1,122 @@ +/** + * \file basic/base/ailayer/ailayer_sigmoid.c + * \version 2.0alpha + * \date 20.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/ailayer/ailayer_sigmoid.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_layertype_t ailayer_sigmoid_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "Sigmoid", + .print_specs = ailayer_sigmoid_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_layertype_t *ailayer_sigmoid_type = &ailayer_sigmoid_type_s; + + +ailayer_t *ailayer_sigmoid(ailayer_sigmoid_t *layer, ailayer_t *input_layer) +{ + layer->base.layer_type = ailayer_sigmoid_type; + + layer->base.input_layer = input_layer; + input_layer->output_layer = &(layer->base); + + layer->base.layer_configuration = layer; + layer->base.result.dtype = layer->dtype; + layer->base.result.dim = input_layer->result.dim; + layer->base.result.shape = input_layer->result.shape; + + layer->base.deltas.dtype = layer->dtype; + layer->base.deltas.dim = 2; + layer->base.deltas.shape = layer->base.result.shape; + + layer->base.forward = ailayer_sigmoid_forward; + layer->base.backward = ailayer_sigmoid_backward; + + layer->base.calc_result_shape = ailayer_sigmoid_calc_result_shape; + layer->base.sizeof_paramem = 0; + layer->base.set_paramem = 0; + layer->base.sizeof_trainmem = 0; + layer->base.set_trainmem = 0; + + layer->base.trainable_params_count = 0; + + return &(layer->base); +} + +void ailayer_sigmoid_forward(ailayer_t *self) +{ + ailayer_sigmoid_t *layer = (ailayer_sigmoid_t *)(self->layer_configuration); + aitensor_t *x_in = &(self->input_layer->result); + aitensor_t *x_out = &(self->result); + + layer->sigmoid(x_in, x_out); + return; +} + + +void ailayer_sigmoid_backward(ailayer_t *self) +{ + ailayer_sigmoid_t *layer = (ailayer_sigmoid_t *)(self->layer_configuration); + aitensor_t *delta_in = &(self->deltas); + aitensor_t *delta_out = &(self->output_layer->deltas); + aitensor_t *x_in = &(self->input_layer->result); + + int8_t temp_result_params[aimath_sizeof_tensor_params(x_in) + 1]; // +1 to prevent array of size 0 + int8_t temp_result_data[aimath_sizeof_tensor_data(x_in)]; + aitensor_t temp_result = { + .dim = 2, + .shape = x_in->shape, + .data = temp_result_data, + .dtype = x_in->dtype, + .tensor_params = temp_result_params + }; + + // delta_in = delta_out .* sigmoid'(x_in) + layer->sigmoid(x_in, &temp_result); + layer->d_sigmoid(&temp_result, &temp_result); + layer->multiply(&temp_result, delta_out, delta_in); + + return; +} + +void ailayer_sigmoid_calc_result_shape(ailayer_t *self) +{ + /* Unused: Shape is already defined (Pointer) + + self->result.shape[0] = self->input_layer->result.shape[0]; + self->result.shape[1] = self->input_layer->result.shape[1]; + */ + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void ailayer_sigmoid_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)) +{ + return; +} +#endif + diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_sigmoid.h b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_sigmoid.h new file mode 100644 index 0000000..1683f96 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_sigmoid.h @@ -0,0 +1,182 @@ +/** + * \file basic/base/ailayer/ailayer_sigmoid.h + * \internal + * \date 20.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link ailayer layer \endlink implementation of the Sigmoid activation layer + * + * This is an "abstract" data-type independent implementation. To use the layer use one of the provided + * implementations for a specific hardware and data-type (for example from ailayer_sigmoid_default.h) or set + * the required math functions on your own. + * + * \image html ailayer_sigmoid_schematic.png width=200px + * + * The Sigmoid layer is used as an activation function layer right after a dense layer. It calculates + * @f[ + * y = \sigma(x) = \frac{1}{1+e^{-x}} + * @f] + * for every element of the input tensor. + * + * The results of the forward pass of this layer are written to the result tensor of the base ailayer_t struct. + */ + +#ifndef SIGMOID_LAYER +#define SIGMOID_LAYER + +#include "core/aifes_core.h" + +typedef struct ailayer_sigmoid ailayer_sigmoid_t; + +/** @brief General \link ailayer_sigmoid.h Sigmoid layer \endlink struct +* +*/ +struct ailayer_sigmoid { + ailayer_t base; /**< Inherited field members from general ailayer struct. */ + const aimath_dtype_t *dtype; /**< Data type of the input and inference result values. */ + + /** @name Math functions + * @brief Required data type specific math functions + */ + ///@{ + + /** @brief Required math function: Sigmoid + * + * Requires a math function that calculates the element wise sigmoid of a tensor:\n + * @f[ + * result_{i} = \sigma(x_{i}) = \frac{1}{1 + e^{-x_{i}}} + * @f] + * + * @param x N-dimensional tensor (input) + * @param result N-dimensional tensor (output) + */ + void (*sigmoid)(const aitensor_t *x, aitensor_t *result); + + /** @brief Required math function: Derivative of sigmoid + * + * Requires a math function that calculates the element wise sigmoid derivative of a tensor:\n + * @f[ + * result_{i} = \sigma'(x_{i}) = \sigma(x_{i}) \cdot (1 - \sigma(x_{i})) + * @f] + * + * @param sigmoid_x N-dimensional tensor with the sigmoid values \f$ \sigma(x_{i}) \f$ (input) + * @param result N-dimensional tensor (output) + */ + void (*d_sigmoid)(const aitensor_t *sigmoid_x, aitensor_t *result); + + /** @brief Required math function: Element wise tensor multiplication + * + * Requires a math function that multiplies two tensors element wise:\n + * @f[ + * result = a \circ b + * @f] + */ + void (*multiply)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + ///@} +}; + +/** @brief Sigmoid layer type + * + * Defines the type of the layer (for example for type checks and debug prints). + * See aicore_layertype for more information about the layer type. + */ +extern const aicore_layertype_t *ailayer_sigmoid_type; + +/** @brief Initialize and connect the given Sigmoid layer + * + * This function represents the "constructor" of the abstract Sigmoid layer. It initializes the layer structure + * and connects it to the previous layer.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example ailayer_sigmoid_f32_default()). + * + * @param *layer The layer to initialize. + * @param *input_layer The previous layer that provides the inputs to the layer. + * @return Pointer to the (successfully) initialized general layer structure (ailayer_sigmoid.base). + */ +ailayer_t *ailayer_sigmoid(ailayer_sigmoid_t *layer, ailayer_t *input_layer); + +/** @brief Calculate the forward pass for given Sigmoid layer + * + * *Implementation of ailayer.forward.* + * + * It uses the result tensor of the previous layer as input and writes the result of the forward pass + * to the result tensor (ailayer.result) of the given layer. + * + * Calculation of the forward pass result: + * @f[ + * x_{out} \leftarrow \sigma(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ x_{out} \f$: Result of the forward pass of this layer\n\n + * + * Used math functions: + * * ailayer_sigmoid.sigmoid + * + * @param *self Layer to calculate the forward path for. + */ +void ailayer_sigmoid_forward(ailayer_t *self); + +/** @brief Calculate the backward pass for the given Sigmoid layer + * + * *Implementation of ailayer.backward.* + * + * It uses the deltas tensor of the next layer as input and writes the result of the backward pass + * to the deltas tensor (ailayer.deltas) of the given layer. + * + * Calculation of the errors for the previous layer: + * @f[ + * \delta_{in} \leftarrow \delta_{out} \circ \sigma'(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ \delta_{in} \f$: Result of the backward pass of this layer\n + * \f$ \delta_{out} \f$: Result of the backward pass of the next layer\n\n + * + * Used math functions: + * * ailayer_sigmoid.sigmoid + * * ailayer_sigmoid.d_sigmoid + * * ailayer_sigmoid.multiply + * + * @param *self Layer to calculate the backward path for. + */ +void ailayer_sigmoid_backward(ailayer_t *self); + +/** @brief Calculate the shape of the result tensor + * + * *Implementation of ailayer.calc_result_shape.* + * + * As the result tensor shape is shared with the result tensor shape of the previous layer (no change in shape is needed), + * this function returns without doing anything. + * + * @param *self Layer to calculate the resulting shape for. + */ +void ailayer_sigmoid_calc_result_shape(ailayer_t *self); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the layer specification + * + * @param *self The layer to print the specification for + * @param *print Pointer to the print function to use + */ +void ailayer_sigmoid_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // SIGMOID_LAYER diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_softmax.c b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_softmax.c new file mode 100644 index 0000000..f7d2ae4 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_softmax.c @@ -0,0 +1,95 @@ +/** + * \file basic/base/ailayer/ailayer_softmax.c + * \version 2.0alpha + * \date 07.12.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/ailayer/ailayer_softmax.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_layertype_t ailayer_softmax_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "Softmax", + .print_specs = ailayer_softmax_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_layertype_t *ailayer_softmax_type = &ailayer_softmax_type_s; + + +ailayer_t *ailayer_softmax(ailayer_softmax_t *layer, ailayer_t *input_layer) +{ + layer->base.layer_type = ailayer_softmax_type; + + layer->base.input_layer = input_layer; + input_layer->output_layer = &(layer->base); + + layer->base.layer_configuration = layer; + layer->base.result.dtype = layer->dtype; + layer->base.result.shape = input_layer->result.shape; + layer->base.result.dim = input_layer->result.dim; + + // No backward pass supported yet + layer->base.deltas.dtype = 0; + + layer->base.forward = ailayer_softmax_forward; + layer->base.backward = 0; + + layer->base.calc_result_shape = ailayer_softmax_calc_result_shape; + layer->base.sizeof_paramem = 0; + layer->base.set_paramem = 0; + layer->base.sizeof_trainmem = 0; + layer->base.set_trainmem = 0; + + layer->base.trainable_params_count = 0; + + return &(layer->base); +} + +void ailayer_softmax_forward(ailayer_t *self) +{ + ailayer_softmax_t *layer = (ailayer_softmax_t *)(self->layer_configuration); + aitensor_t *x_in = &(self->input_layer->result); + aitensor_t *x_out = &(self->result); + + layer->softmax(x_in, x_out); + return; +} + +void ailayer_softmax_calc_result_shape(ailayer_t *self) +{ + /* Unused: Shape is already defined (Pointer) + + self->result.shape[0] = self->input_layer->result.shape[0]; + self->result.shape[1] = self->input_layer->result.shape[1]; + */ + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void ailayer_softmax_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)) +{ + return; +} +#endif + diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_softmax.h b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_softmax.h new file mode 100644 index 0000000..9b97ec2 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_softmax.h @@ -0,0 +1,140 @@ + +/** + * \file basic/base/ailayer/ailayer_softmax.h + * \internal + * \date 07.12.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link ailayer layer \endlink implementation of the Softmax activation layer + * + * This is an "abstract" data-type independent implementation. To use the layer use one of the provided + * implementations for a specific hardware and data-type (for example from ailayer_softmax_default.h) or set + * the required math functions on your own.\n + * This layer is designed as an output-layer and has no backward implementation. If you want to train an ANN with this layer, + * use the \link ailoss_crossentropy.h Cross-Entropy loss \endlink which has a combined Softmax-Cross-Entropy backward + * function that is faster to compute! + * + * \image html ailayer_softmax_schematic.png width=200px + * + * The Softmax layer is used as an activation function layer right after a dense layer. It calculates + * @f[ + * y_i = \frac{e^{x_i}}{\sum_{j=1}^{K} e^{x_j}} + * @f] + * for every element of the input tensor with \f$ K \f$ elements. + * + * The results of the forward pass of this layer are written to the result tensor of the base ailayer_t struct. + */ + +#ifndef SOFTMAX_LAYER +#define SOFTMAX_LAYER + +#include "core/aifes_core.h" + +typedef struct ailayer_softmax ailayer_softmax_t; + +/** @brief General \link ailayer_softmax.h Softmax layer \endlink struct +* +*/ +struct ailayer_softmax { + ailayer_t base; /**< Inherited field members from general ailayer struct. */ + const aimath_dtype_t *dtype; /**< Data type of the input and inference result values. */ + + /** @name Math functions + * @brief Required data type specific math functions + */ + ///@{ + + /** @brief Required math function: Softmax + * + * Requires a math function that calculates the element wise softmax of a tensor:\n + * @f[ + * result_{i} = \frac{e^{x_i}}{\sum_{j=1}^{K} e^{x_j}} + * @f] + * + * @param x N-dimensional tensor (input) + * @param result N-dimensional tensor (output) + */ + void (*softmax)(const aitensor_t *x, aitensor_t *result); + + ///@} +}; + +/** @brief Softmax layer type + * + * Defines the type of the layer (for example for type checks and debug prints). + * See aicore_layertype for more information about the layer type. + */ +extern const aicore_layertype_t *ailayer_softmax_type; + +/** @brief Initialize and connect the given Softmax layer + * + * This function represents the "constructor" of the abstract Softmax layer. It initializes the layer structure + * and connects it to the previous layer.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example ailayer_softmax_f32_default()). + * + * @param *layer The layer to initialize. + * @param *input_layer The previous layer that provides the inputs to the layer. + * @return Pointer to the (successfully) initialized general layer structure (ailayer_softmax.base). + */ +ailayer_t *ailayer_softmax(ailayer_softmax_t *layer, ailayer_t *input_layer); + +/** @brief Calculate the forward pass for given Softmax layer + * + * *Implementation of ailayer.forward.* + * + * It uses the result tensor of the previous layer as input and writes the result of the forward pass + * to the result tensor (ailayer.result) of the given layer. + * + * Calculation of the forward pass result: + * @f[ + * x_{out} \leftarrow Softmax(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ x_{out} \f$: Result of the forward pass of this layer\n\n + * + * Used math functions: + * * ailayer_softmax.softmax + * + * @param *self Layer to calculate the forward path for. + */ +void ailayer_softmax_forward(ailayer_t *self); + +/** @brief Calculate the shape of the result tensor + * + * *Implementation of ailayer.calc_result_shape.* + * + * As the result tensor shape is shared with the result tensor shape of the previous layer (no change in shape is needed), + * this function returns without doing anything. + * + * @param *self Layer to calculate the resulting shape for. + */ +void ailayer_softmax_calc_result_shape(ailayer_t *self); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the layer specification + * + * @param *self The layer to print the specification for + * @param *print Pointer to the print function to use + */ +void ailayer_softmax_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // SOFTMAX_LAYER diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_softsign.c b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_softsign.c new file mode 100644 index 0000000..9cc1152 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_softsign.c @@ -0,0 +1,120 @@ +/** + * \file basic/base/ailayer/ailayer_softsign.c + * \version 2.0alpha + * \date 16.03.2021 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief A softsign activation function. + * \details + */ + +#include "basic/base/ailayer/ailayer_softsign.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_layertype_t ailayer_softsign_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "Softsign", + .print_specs = ailayer_softsign_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_layertype_t *ailayer_softsign_type = &ailayer_softsign_type_s; + + +ailayer_t *ailayer_softsign(ailayer_softsign_t *layer, ailayer_t *input_layer) +{ + layer->base.layer_type = ailayer_softsign_type; + + layer->base.input_layer = input_layer; + input_layer->output_layer = &(layer->base); + + layer->base.layer_configuration = layer; + layer->base.result.dtype = layer->dtype; + + layer->base.result.dim = input_layer->result.dim; + layer->base.result.shape = input_layer->result.shape; + + layer->base.deltas.dtype = layer->dtype; + layer->base.deltas.dim = 2; + layer->base.deltas.shape = layer->base.result.shape; + + layer->base.forward = ailayer_softsign_forward; + layer->base.backward = ailayer_softsign_backward; + + layer->base.calc_result_shape = ailayer_softsign_calc_result_shape; + layer->base.sizeof_paramem = 0; + layer->base.set_paramem = 0; + layer->base.sizeof_trainmem = 0; + layer->base.set_trainmem = 0; + + layer->base.trainable_params_count = 0; + + return &(layer->base); +} + +void ailayer_softsign_backward(ailayer_t *self) +{ + ailayer_softsign_t *layer = (ailayer_softsign_t *)(self->layer_configuration); + aitensor_t *delta_in = &(self->deltas); + aitensor_t *delta_out = &(self->output_layer->deltas); + aitensor_t *x_in = &(self->input_layer->result); + + int8_t temp_result_params[aimath_sizeof_tensor_params(x_in) + 1]; // +1 to prevent array of size 0 + int8_t temp_result_data[aimath_sizeof_tensor_data(x_in)]; + aitensor_t temp_result = { + .dim = 2, + .shape = x_in->shape, + .data = temp_result_data, + .dtype = x_in->dtype, + .tensor_params = temp_result_params + }; + + layer->softsign(x_in, &temp_result); + layer->d_softsign(&temp_result, &temp_result); + layer->multiply(&temp_result, delta_out, delta_in); + + return; +} + +void ailayer_softsign_forward(ailayer_t *self) +{ + ailayer_softsign_t *layer = (ailayer_softsign_t *)(self->layer_configuration); + aitensor_t *x_in = &(self->input_layer->result); + aitensor_t *x_out = &(self->result); + + layer->softsign(x_in, x_out); + return; +} + +void ailayer_softsign_calc_result_shape(ailayer_t *self) +{ + /* Unused: Shape is already defined (Pointer) + + self->result.shape[0] = self->input_layer->result.shape[0]; + self->result.shape[1] = self->input_layer->result.shape[1]; + */ + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void ailayer_softsign_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)) +{ + return; +} +#endif diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_softsign.h b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_softsign.h new file mode 100644 index 0000000..01524b8 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_softsign.h @@ -0,0 +1,183 @@ + +/** + * \file basic/base/ailayer/ailayer_softsign.h + * \internal + * \date 16.03.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link ailayer layer \endlink implementation of the Softsign activation layer + * + * This is an "abstract" data-type independent implementation. To use the layer use one of the provided + * implementations for a specific hardware and data-type (for example from ailayer_softsign_default.h) or set + * the required math functions on your own. + * + * \image html ailayer_softsign_schematic.png width=200px + * + * The Softsign layer is used as an activation function layer right after a dense layer. It calculates + * @f[ + * y = \frac {x} {1 + |x|} + * @f] + * for every element of the input tensor. + * + * The results of the forward pass of this layer are written to the result tensor of the base ailayer_t struct. + */ + +#ifndef SOFTSIGN_LAYER +#define SOFTSIGN_LAYER + +#include "core/aifes_core.h" + +typedef struct ailayer_softsign ailayer_softsign_t; + +/** @brief General \link ailayer_softsign.h Softsign layer \endlink struct +* +*/ +struct ailayer_softsign { + ailayer_t base; /**< Inherited field members from general ailayer struct. */ + const aimath_dtype_t *dtype; /**< Data type of the input and inference result values. */ + + /** @name Math functions + * @brief Required data type specific math functions + */ + ///@{ + + /** @brief Required math function: Softsign + * + * Requires a math function that calculates the element wise softsign of a tensor:\n + * @f[ + * result_{i} = \frac {x_i} {1 + |x_i|} + * @f] + * + * @param x N-dimensional tensor (input) + * @param result N-dimensional tensor (output) + */ + void (*softsign)(const aitensor_t *x, aitensor_t *result); + + /** @brief Required math function: Derivative of softsign + * + * Requires a math function that calculates the element wise softsign derivative of a tensor:\n + * @f[ + * result_{i} = \frac {x_i} {(1 + |x_i|)^2} + * @f] + * + * @param x N-dimensional tensor (input) + * @param result N-dimensional tensor (output) + */ + void (*d_softsign)(const aitensor_t *softsign_x, aitensor_t *result); + + /** @brief Required math function: Element wise tensor multiplication + * + * Requires a math function that multiplies two tensors element wise:\n + * @f[ + * result = a \circ b + * @f] + */ + void (*multiply)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + ///@} +}; + +/** @brief Softsign layer type + * + * Defines the type of the layer (for example for type checks and debug prints). + * See aicore_layertype for more information about the layer type. + */ +extern const aicore_layertype_t *ailayer_softsign_type; + +/** @brief Initialize and connect the given Softsign layer + * + * This function represents the "constructor" of the abstract Softsign layer. It initializes the layer structure + * and connects it to the previous layer.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example ailayer_softsign_f32_default()). + * + * @param *layer The layer to initialize. + * @param *input_layer The previous layer that provides the inputs to the layer. + * @return Pointer to the (successfully) initialized general layer structure (ailayer_softsign.base). + */ +ailayer_t *ailayer_softsign(ailayer_softsign_t *layer, ailayer_t *input_layer); + +/** @brief Calculate the forward pass for given Tanh layer + * + * *Implementation of ailayer.forward.* + * + * It uses the result tensor of the previous layer as input and writes the result of the forward pass + * to the result tensor (ailayer.result) of the given layer. + * + * Calculation of the forward pass result: + * @f[ + * x_{out} \leftarrow Softsign(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ x_{out} \f$: Result of the forward pass of this layer\n\n + * + * Used math functions: + * * ailayer_softsign.softsign + * + * @param *self Layer to calculate the forward pass for. + */ +void ailayer_softsign_forward(ailayer_t *self); + +/** @brief Calculate the backward pass for the given Softsign layer + * + * *Implementation of ailayer.backward.* + * + * It uses the deltas tensor of the next layer as input and writes the result of the backward pass + * to the deltas tensor (ailayer.deltas) of the given layer. + * + * Calculation of the errors for the previous layer: + * @f[ + * \delta_{in} \leftarrow \delta_{out} \circ Softsign'(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ \delta_{in} \f$: Result of the backward pass of this layer\n + * \f$ \delta_{out} \f$: Result of the backward pass of the next layer\n\n + * + * Used math functions: + * * ailayer_softsign.softsign + * * ailayer_softsign.d_softsign + * * ailayer_softsign.multiply + * + * @param *self Layer to calculate the backward path for. + */ +void ailayer_softsign_backward(ailayer_t *self); + +/** @brief Calculate the shape of the result tensor + * + * *Implementation of ailayer.calc_result_shape.* + * + * As the result tensor shape is shared with the result tensor shape of the previous layer (no change in shape is needed), + * this function returns without doing anything. + * + * @param *self Layer to calculate the resulting shape for. + */ +void ailayer_softsign_calc_result_shape(ailayer_t *self); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the layer specification + * + * @param *self The layer to print the specification for + * @param *print Pointer to the print function to use + */ +void ailayer_softsign_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // SOFTSIGN_LAYER diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_tanh.c b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_tanh.c new file mode 100644 index 0000000..ed88e2b --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_tanh.c @@ -0,0 +1,123 @@ + +/** + * \file basic/base/ailayer/ailayer_tanh.c + * \version 2.0alpha + * \date 17.03.2021 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/ailayer/ailayer_tanh.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_layertype_t ailayer_tanh_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "Tanh", + .print_specs = ailayer_tanh_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_layertype_t *ailayer_tanh_type = &ailayer_tanh_type_s; + + +ailayer_t *ailayer_tanh(ailayer_tanh_t *layer, ailayer_t *input_layer) +{ + layer->base.layer_type = ailayer_tanh_type; + + layer->base.input_layer = input_layer; + input_layer->output_layer = &(layer->base); + + layer->base.layer_configuration = layer; + layer->base.result.dtype = layer->dtype; + layer->base.result.dim = input_layer->result.dim; + layer->base.result.shape = input_layer->result.shape; + + layer->base.deltas.dtype = layer->dtype; + layer->base.deltas.dim = 2; + layer->base.deltas.shape = layer->base.result.shape; + + layer->base.forward = ailayer_tanh_forward; + layer->base.backward = ailayer_tanh_backward; + + layer->base.calc_result_shape = ailayer_tanh_calc_result_shape; + layer->base.sizeof_paramem = 0; + layer->base.set_paramem = 0; + layer->base.sizeof_trainmem = 0; + layer->base.set_trainmem = 0; + + layer->base.trainable_params_count = 0; + + return &(layer->base); +} + +void ailayer_tanh_forward(ailayer_t *self) +{ + ailayer_tanh_t *layer = (ailayer_tanh_t *)(self->layer_configuration); + aitensor_t *x_in = &(self->input_layer->result); + aitensor_t *x_out = &(self->result); + + layer->tanh(x_in, x_out); + return; +} + + +void ailayer_tanh_backward(ailayer_t *self) +{ + ailayer_tanh_t *layer = (ailayer_tanh_t *)(self->layer_configuration); + aitensor_t *delta_in = &(self->deltas); + aitensor_t *delta_out = &(self->output_layer->deltas); + aitensor_t *x_in = &(self->input_layer->result); + + int8_t temp_result_params[aimath_sizeof_tensor_params(x_in) + 1]; // +1 to prevent array of size 0 + int8_t temp_result_data[aimath_sizeof_tensor_data(x_in)]; + aitensor_t temp_result = { + .dim = 2, + .shape = x_in->shape, + .data = temp_result_data, + .dtype = x_in->dtype, + .tensor_params = temp_result_params + }; + + // delta_in = delta_out .* tanh'(x_in) + layer->tanh(x_in, &temp_result); + layer->d_tanh(&temp_result, &temp_result); + layer->multiply(&temp_result, delta_out, delta_in); + + return; +} + +void ailayer_tanh_calc_result_shape(ailayer_t *self) +{ + /* Unused: Shape is already defined (Pointer) + + self->result.shape[0] = self->input_layer->result.shape[0]; + self->result.shape[1] = self->input_layer->result.shape[1]; + */ + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void ailayer_tanh_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)) +{ + return; +} +#endif + diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_tanh.h b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_tanh.h new file mode 100644 index 0000000..177aef5 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_tanh.h @@ -0,0 +1,182 @@ + +/** + * \file basic/base/ailayer/ailayer_tanh.h + * \internal + * \date 17.03.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link ailayer layer \endlink implementation of the Tanh activation layer + * + * This is an "abstract" data-type independent implementation. To use the layer use one of the provided + * implementations for a specific hardware and data-type (for example from ailayer_tanh_default.h) or set + * the required math functions on your own. + * + * \image html ailayer_tanh_schematic.png width=200px + * + * The Tanh layer is used as an activation function layer right after a dense layer. It calculates + * @f[ + * y = \tanh(x) = \frac{e^x - e^{-x}}{e^x + e^{-x}} + * @f] + * for every element of the input tensor. + * + * The results of the forward pass of this layer are written to the result tensor of the base ailayer_t struct. + */ +#ifndef TANH_LAYER +#define TANH_LAYER + +#include "core/aifes_core.h" + +typedef struct ailayer_tanh ailayer_tanh_t; + +/** @brief General \link ailayer_tanh.h Tanh layer \endlink struct +* +*/ +struct ailayer_tanh { + ailayer_t base; /**< Inherited field members from general ailayer struct. */ + const aimath_dtype_t *dtype; /**< Data type of the input and inference result values. */ + + /** @name Math functions + * @brief Required data type specific math functions + */ + ///@{ + + /** @brief Required math function: Tanh + * + * Requires a math function that calculates the element wise tanh of a tensor:\n + * @f[ + * result_{i} = \tanh(x_{i}) = \frac{e^{x_i} - e^{-x_i}}{e^{x_i} + e^{-x_i}} + * @f] + * + * @param x N-dimensional tensor (input) + * @param result N-dimensional tensor (output) + */ + void (*tanh)(const aitensor_t *x, aitensor_t *result); + + /** @brief Required math function: Derivative of tanh + * + * Requires a math function that calculates the element wise tanh derivative of a tensor:\n + * @f[ + * result_{i} = tanh'(x_{i}) = 1 - tanh(x_{i})^2 + * @f] + * + * @param tanh_x N-dimensional tensor with the tanh values \f$ tanh(x_{i}) \f$ (input) + * @param result N-dimensional tensor (output) + */ + void (*d_tanh)(const aitensor_t *tanh_x, aitensor_t *result); + + /** @brief Required math function: Element wise tensor multiplication + * + * Requires a math function that multiplies two tensors element wise:\n + * @f[ + * result = a \circ b + * @f] + */ + void (*multiply)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + ///@} +}; + +/** @brief Tanh layer type + * + * Defines the type of the layer (for example for type checks and debug prints). + * See aicore_layertype for more information about the layer type. + */ +extern const aicore_layertype_t *ailayer_tanh_type; + +/** @brief Initialize and connect the given Tanh layer + * + * This function represents the "constructor" of the abstract Tanh layer. It initializes the layer structure + * and connects it to the previous layer.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example ailayer_tanh_f32_default()). + * + * @param *layer The layer to initialize. + * @param *input_layer The previous layer that provides the inputs to the layer. + * @return Pointer to the (successfully) initialized general layer structure (ailayer_tanh.base). + */ +ailayer_t *ailayer_tanh(ailayer_tanh_t *layer, ailayer_t *input_layer); + +/** @brief Calculate the forward pass for given Tanh layer + * + * *Implementation of ailayer.forward.* + * + * It uses the result tensor of the previous layer as input and writes the result of the forward pass + * to the result tensor (ailayer.result) of the given layer. + * + * Calculation of the forward pass result: + * @f[ + * x_{out} \leftarrow tanh(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ x_{out} \f$: Result of the forward pass of this layer\n\n + * + * Used math functions: + * * ailayer_tanh.tanh + * + * @param *self Layer to calculate the forward path for. + */ +void ailayer_tanh_forward(ailayer_t *self); + +/** @brief Calculate the backward pass for the given Tanh layer + * + * *Implementation of ailayer.backward.* + * + * It uses the deltas tensor of the next layer as input and writes the result of the backward pass + * to the deltas tensor (ailayer.deltas) of the given layer. + * + * Calculation of the errors for the previous layer: + * @f[ + * \delta_{in} \leftarrow \delta_{out} \circ tanh'(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ \delta_{in} \f$: Result of the backward pass of this layer\n + * \f$ \delta_{out} \f$: Result of the backward pass of the next layer\n\n + * + * Used math functions: + * * ailayer_tanh.tanh + * * ailayer_tanh.d_tanh + * * ailayer_tanh.multiply + * + * @param *self Layer to calculate the backward path for. + */ +void ailayer_tanh_backward(ailayer_t *self); + +/** @brief Calculate the shape of the result tensor + * + * *Implementation of ailayer.calc_result_shape.* + * + * As the result tensor shape is shared with the result tensor shape of the previous layer (no change in shape is needed), + * this function returns without doing anything. + * + * @param *self Layer to calculate the resulting shape for. + */ +void ailayer_tanh_calc_result_shape(ailayer_t *self); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the layer specification + * + * @param *self The layer to print the specification for + * @param *print Pointer to the print function to use + */ +void ailayer_tanh_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // TANH_LAYER diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_template.c b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_template.c new file mode 100644 index 0000000..2c39b9b --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_template.c @@ -0,0 +1,211 @@ +/** + * \file basic/base/ailayer/ailayer_template.c + * \version 2.0alpha + * \date 20.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/ailayer/ailayer_template.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_layertype_t ailayer_template_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "Template", + .print_specs = ailayer_template_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_layertype_t *ailayer_template_type = &ailayer_template_type_s; + +ailayer_t *ailayer_template(ailayer_template_t *layer, ailayer_t *input_layer) +{ + // Connect the layer with its input + layer->base.input_layer = input_layer; + input_layer->output_layer = &(layer->base); + + // Init basic layer variables + layer->base.layer_configuration = layer; + layer->base.result.dtype = layer->dtype; + layer->base.result.shape = layer->result_shape; + layer->base.result.dim = 2; + + layer->base.deltas.dtype = layer->dtype; + layer->base.deltas.shape = input_layer->result.shape; + layer->base.deltas.dim = 2; + + // Set the function pointers + layer->base.forward = ailayer_template_forward; + layer->base.backward = ailayer_template_backward; + + layer->base.calc_result_shape = ailayer_template_calc_result_shape; + layer->base.sizeof_paramem = ailayer_template_sizeof_paramem; + layer->base.set_paramem = ailayer_template_set_paramem; + layer->base.sizeof_paramem = ailayer_template_sizeof_trainmem; + layer->base.set_trainmem = ailayer_template_set_trainmem; + + return &layer->base; +} + +/* +* Layer naming: +* ------- +* x_in -> | L | -> x_out +* | A | +* | Y | +* | E | +* delta_in <- | R | <- delta_out +* ------- +*/ + + +void ailayer_template_forward(ailayer_t *self) +{ + ailayer_template_t *layer = (ailayer_template_t *)(self->layer_configuration); + aitensor_t *x_in = &(self->input_layer->result); + aitensor_t *x_out = &(self->result); + + // Example forward: x_out = x_in + params + layer->tensor_add(x_in, layer->params, x_out); + + return; +} + +void ailayer_template_backward(ailayer_t *self) +{ + ailayer_template_t *layer = (ailayer_template_t *)(self->layer_configuration); + aitensor_t *x_in = &(self->input_layer->result); +// aitensor_t *x_out = &(self->result); + aitensor_t *delta_in = &(self->deltas); + aitensor_t *delta_out = &(self->output_layer->deltas); + + // Temporary result for calculation can be created here. Document the needed amount of temp memory! + // Remove code if unused! + float temp_result_data[delta_out->shape[0] * delta_out->shape[1]]; + uint16_t temp_result_shape[] = {delta_out->shape[0], delta_out->shape[1]}; + aitensor_t temp_result = { + .dim = 2, + .shape = temp_result_shape, + .data = temp_result_data, + .dtype = x_in->dtype + }; + + // 1) Calculate the gradients and add to the gradients tensor + layer->copy_tensor(delta_out, &temp_result); // (Unnecessary, just show the use of temp_result) + layer->tensor_add(layer->d_params, &temp_result, layer->d_params); + + // 2) Calculate delta for next layer. Do not before calculating gradients!!! May override x_in!!! + layer->copy_tensor(&temp_result, delta_in); + + return; +} + +void ailayer_template_calc_result_shape(ailayer_t *self) +{ +// ailayer_template_t *layer = (ailayer_template_t *)(self->layer_configuration); + aitensor_t *x_in = &(self->input_layer->result); + aitensor_t *x_out = &(self->result); + + x_out->shape[0] = x_in->shape[0]; + x_out->shape[1] = x_in->shape[1]; + + return; +} + +uint32_t ailayer_template_sizeof_paramem(const ailayer_t *self) +{ + ailayer_template_t *layer = (ailayer_template_t *)(self->layer_configuration); +// const aitensor_t *x_in = &(self->input_layer->result); + const aitensor_t *x_out = &(self->result); + + uint32_t memory = 0; + + // Memory amount for params. Attention: result shape is calculated but params tensor is not available yet. + memory += sizeof(aitensor_t); // struct + memory += 2 * sizeof(uint16_t); // shape array + memory += x_out->shape[0] * x_out->shape[1] * aimath_sizeof_dtype(layer->dtype); // data + + return memory; +} + +void ailayer_template_set_paramem(ailayer_t *self, void *memory_ptr) +{ + ailayer_template_t *layer = (ailayer_template_t *)(self->layer_configuration); + aitensor_t *x_in = &(self->input_layer->result); +// aitensor_t *x_out = &(self->result); + + uint32_t address_counter = 0; + + // Params memory distribution + // tensor struct: + layer->params = memory_ptr + address_counter; + address_counter += sizeof(aitensor_t); + layer->params->dim = 2; + layer->params->dtype = layer->dtype; + // shape array: + layer->params->shape = memory_ptr + address_counter; + address_counter += 2 * sizeof(uint16_t); + layer->params->shape[0] = x_in->shape[0]; + layer->params->shape[1] = x_in->shape[1]; + // data: + layer->params->data = memory_ptr + address_counter; + address_counter += aimath_sizeof_tensor_data(layer->params); + + return; +} + +uint32_t ailayer_template_sizeof_trainmem(const ailayer_t *self) +{ + ailayer_template_t *layer = (ailayer_template_t *)(self->layer_configuration); + + uint32_t memory = 0; + + // Sum up the memory sizes for every parameter tensor + memory += aimath_sizeof_tensor(layer->params); // Size of tensor struct and data but not the shape array, because shape array is shared with params + + return memory; +} + +void ailayer_template_set_trainmem(ailayer_t *self, void *memory_ptr) +{ + ailayer_template_t *layer = (ailayer_template_t *)(self->layer_configuration); + + uint32_t address_counter = 0; + + // Params gradient memory distribution + // tensor struct: + layer->d_params = memory_ptr + address_counter; + address_counter += sizeof(aitensor_t); + layer->d_params->dim = 2; + layer->d_params->shape = layer->params->shape; // shared shape + // data: + layer->d_params->data = memory_ptr + address_counter; + address_counter += aimath_sizeof_tensor_data(layer->d_params); + + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void ailayer_template_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)) +{ + return; +} +#endif diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_template.h b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_template.h new file mode 100644 index 0000000..c5f28d4 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailayer/ailayer_template.h @@ -0,0 +1,131 @@ +/** + * \file basic/base/ailayer/ailayer_template.h + * \internal + * \date 20.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief [DEPRECATED] This file acts as a reference for new layer structures. + * Some of the shown functions are mandatory, some are optional. + * \details + */ + +#ifndef TEMPLATE_LAYER +#define TEMPLATE_LAYER + +#include "core/aifes_core.h" + +typedef struct ailayer_template ailayer_template_t; + +/** @brief Template for general AIfES layers +* +*/ +struct ailayer_template { + ailayer_t base; /**< Inherited field members from general ailayer struct. */ // The base or super "class" of the layer. + const aimath_dtype_t *dtype; /**< Datatype of the inference result values. */ + + // Configurations of the layer, that can be configured by the user when creating the layer + uint32_t example_configuration; + + // Tensor pointer for trainable parameters [DEPRECATED] + aitensor_t *params; + aitensor_t *d_params; // Gradients + + // If the shape of the result differs from the input shape, this array can be used for the result. + // If not, the result shape array of the input tensor can be used for the result tensor too. + uint16_t result_shape[2]; /**< Inference result tensor shape (n x m matrix). */ + + aitensor_t *trainable_params[2]; /**< Pointer to the weights and biases (which are the trainable parameters). */ + aitensor_t *gradients[2]; /**< Gradients structure for the back propagation algorithm. */ + void *optimem[2]; /**< Memory field used by the trainings optimizer. */ + + // Make the math functions accessable for expert developers + void (*copy_tensor)(const aitensor_t *from, aitensor_t *to); /**< Function pointer to the used tensor copy function. */ + void (*tensor_add)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); /**< Function pointer to the used tensor addition. */ +}; + +extern const aicore_layertype_t *ailayer_template_type; +/** @brief Initialize the given layer struct - __mandatory__ + * + * Initializes and sets all parameters needed by the layer structure. + * + * @param *layer The layer to initialize. + * @param *input_layer The layer that provides the inputs to the initialized layer. + * @return Pointer to the (successfully) initialized layer structure. + */ + +ailayer_t *ailayer_template(ailayer_template_t *layer, ailayer_t *input_layer); + +/** @brief Calculate the forward path for given layer - __mandatory__ + * + * @param *self Layer to calculate the forward path for. + */ +void ailayer_template_forward(ailayer_t *self); + +/** @brief Calculate the backward (training) path for the given layer - __depends on usability in training__ + * + * @param *self Layer to calculate the backward path for. + */ +void ailayer_template_backward(ailayer_t *self); + +/** @brief Calculate the shape of the result tensor + * + * @param *self Layer to calculate the resulting shape for. + */ +void ailayer_template_calc_result_shape(ailayer_t *self); + +/** @brief Calculate and return the parameter memory size + * needed for this layer - __mandatory for hidden layers__ + * + * @param *self The layer to calculate the parameter memory size for + * @return Calculated parameter memory size in bytes. + */ +uint32_t ailayer_template_sizeof_paramem(const ailayer_t *self); + +/** @brief Distribute provided memory to the parameter pointers - __mandatory for hidden layers__ + * + * @param *self The layer to set the memory fields for. + * @param *memory_ptr Pointer to the buffer provided for the layer parameters. + */ +void ailayer_template_set_paramem(ailayer_t *self, void *memory_ptr); + +/** @brief Calculate and return the necessary memory size + * needed by this layer for training - __mandatory for hidden layers__ + * + * @param *self The layer to calculate the gradient memory size for. + * @return Calculated gradient memory size in bytes. + */ +uint32_t ailayer_template_sizeof_trainmem(const ailayer_t *self); + +/** @brief Distribute provided memory to the training gradients pointers - __mandatory for hidden layers__ + * + * @param *self The layer to set the memory fields for. + * @param *memory_ptr Pointer to the buffer provided for the layer gradients. + */ +void ailayer_template_set_trainmem(ailayer_t *self, void *memory_ptr); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the layer specification + * + * @param *self The layer to print the specification for + * @param *print Pointer to the print function to use + */ +void ailayer_template_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)); +#endif + +#endif // TEMPLATE_LAYER diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailoss/ailoss_crossentropy.c b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailoss/ailoss_crossentropy.c new file mode 100644 index 0000000..1591d70 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailoss/ailoss_crossentropy.c @@ -0,0 +1,110 @@ +/** + * \file basic/base/ailoss/ailoss_crossentropy.c + * \version 2.0alpha + * \date 12.01.2021 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/ailoss/ailoss_crossentropy.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_losstype_t ailoss_crossentropy_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "Cross-entropy", + .print_specs = ailoss_crossentropy_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_losstype_t *ailoss_crossentropy_type = &ailoss_crossentropy_type_s; + +ailoss_t *ailoss_crossentropy(ailoss_crossentropy_t *loss, ailayer_t *input_layer) +{ + loss->base.loss_type = ailoss_crossentropy_type; + + loss->base.connection_layer.input_layer = input_layer; + input_layer->output_layer = &(loss->base.connection_layer); + + loss->base.loss_configuration = loss; + + loss->base.connection_layer.deltas.dtype = loss->dtype; + + loss->base.calc_delta = ailoss_crossentropy_calc_delta; + loss->base.calc_loss = ailoss_crossentropy_calc_loss; + + // Check for valid input and override the backward function (not needed) + if(input_layer->layer_type == ailayer_softmax_type || input_layer->layer_type == ailayer_sigmoid_type){ + input_layer->backward = ailoss_crossentropy_dummy_backward; + } + else{ + #ifdef AIDEBUG_PRINT_ERROR_MESSAGES + printf("\n!!! ERROR !!! (ailoss_crossentropy): No valid input layer. Use either Sigmoid or Softmax as input.\n"); + #endif + return 0; + } + + return &loss->base; +} + +void ailoss_crossentropy_calc_delta(ailoss_t *self, const aitensor_t *target_data) +{ + ailoss_crossentropy_t *loss = (ailoss_crossentropy_t *)(self->loss_configuration); + aitensor_t *predicted_data = &(self->connection_layer.input_layer->result); + aitensor_t *deltas = &(self->connection_layer.input_layer->deltas); + + deltas->shape = predicted_data->shape; // ToDo: Maybe remove or put it in constructor + + // Calc dC/dz directly instead of dC/da_L and then da_L/dz in the output layer + // dC/dz = (a_L - y) ( * 1/batch_size) + loss->tensor_sub(predicted_data, target_data, deltas); + + // ToDo: Scale by batch size. (Divide / Shift) + + return; +} + +void ailoss_crossentropy_calc_loss(ailoss_t *self, const aitensor_t *target_data, void *result) +{ + ailoss_crossentropy_t *loss = (ailoss_crossentropy_t *)(self->loss_configuration); + + aitensor_t *predicted_data = &(self->connection_layer.input_layer->result); + + //C = - y * log(a_L) ( * 1/batch_size) + loss->crossentropy(predicted_data, target_data, result); + + // ToDo: Scale by batch size. (Divide / Shift) + + return; +} + +void ailoss_crossentropy_dummy_backward(ailayer_t *self) +{ + return; +} + + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void ailoss_crossentropy_print_specs(const ailoss_t *self, int (*print)(const char *format, ...)) +{ + return; +} +#endif + diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailoss/ailoss_crossentropy.h b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailoss/ailoss_crossentropy.h new file mode 100644 index 0000000..f8cb30e --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailoss/ailoss_crossentropy.h @@ -0,0 +1,205 @@ +/** + * \file basic/base/ailoss/ailoss_crossentropy.h + * \internal + * \date 12.01.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link ailoss loss \endlink implementation of the Cross-Entropy loss + * + * This is an "abstract" data-type independent implementation. To use the loss, use one of the provided + * implementations for a specific hardware and data-type (for example from ailoss_crossentropy_default.h) or set + * the required math functions on your own. + * + * The Cross-Entropy loss ist best suitable for classification tasks + * and works only with \link ailayer_sigmoid.h Sigmoid \endlink or \link ailayer_softmax.h Softmax \endlink output layers. + * + * For **binary classification** (binary targets 0 or 1) with **Sigmoid output layer**, the loss / cost is calculated as + * @f[ + * L(y, \hat{y}) = - \sum_{i=0}^{N} (y_i \log(\hat{y}_i) + (1-y) \log(1-\hat{y}_i)) + * @f] + * with the predicted values \f$ \hat{y}_i \f$ and the target values \f$ y_i \f$. + * \f$ N \f$ is the number of elements of the \f$ y \f$ tensor. + * + * For **categorigal classification** (one-hot encoded targets) with **Softmax output layer**, the loss / cost is calculated as + * @f[ + * L(y, \hat{y}) = - \sum_{i=0}^{N} y_{i} \log(\hat{y}_{i}) + * @f] + * with the predicted values \f$ \hat{y}_i \f$ and the target values \f$ y_i \f$. + * \f$ N \f$ is the number of elements of the \f$ y \f$ tensor. + * + * To get the "mean" normalization, you have to modify the learning rate to \f$ lr = \frac {1}{o \cdot n} \cdot lr \f$ + * with the number of outputs \f$ o \f$ and the batch size \f$ n \f$. + * + * The loss can be calculated with ailoss_crossentropy_calc_loss(). For training the deltas /errors on the target data are + * calculated with ailoss_crossentropy_calc_delta() and written to the deltas tensor of the connection layer. + */ + +#ifndef CROSSENTROPY_LOSS +#define CROSSENTROPY_LOSS + +#include "core/aifes_core.h" +#include "basic/base/ailayer/ailayer_softmax.h" +#include "basic/base/ailayer/ailayer_sigmoid.h" + +typedef struct ailoss_crossentropy ailoss_crossentropy_t; /**< New data type name for code reduction. */ + +/** @brief General \link ailoss_crossentropy.h Cross-Entropy loss \endlink struct +* +*/ +struct ailoss_crossentropy { + ailoss_t base; /**< Inherited field members from general ailoss struct. */ + const aimath_dtype_t *dtype; /**< Main data type of the loss. */ + + /** @name Math functions + * @brief Required data type specific math functions + */ + ///@{ + + /** @brief Required math function: Element wise tensor subtraction + * + * Requires a math function that subtracts two tensors element wise:\n + * @f[ + * result = a - b + * @f] + */ + void (*tensor_sub)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + /** @brief Required math function: Cross-Entropy between two tensors + * + * Requires a math function that calculates the Cross-Entropy between two tensors.\n + * For a Sigmoid output layer it should be the binary Cross-Entropy: + * @f[ + * result = -\sum (target \cdot \log(predicted) + (1 - target) \log(1 - predicted)) + * @f] + * + * For a Softmax output layer it should be the categorical Cross-Entropy: + * @f[ + * result = -\sum target \cdot \log(predicted) + * @f] + * + * @param result A Scalar of the defined data type + */ + void (*crossentropy)(const aitensor_t *predicted_data, const aitensor_t *target_data, void *result); + + ///@} +}; + +/** @brief Cross-Entropy loss type + * + * Defines the type of the loss (for example for type checks and debug prints). + * See aicore_losstype for more information about the loss type. + */ +extern const aicore_losstype_t *ailoss_crossentropy_type; + +/** @brief Initialize and connect the given Cross-Entropy loss + * + * This function represents the "constructor" of the abstract Cross-Entropy loss. It initializes the loss structure + * and connects it to the output layer of the AIfES model.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example ailoss_crossentropy_f32_default()). + * + * @param *loss The loss to initialize. + * @param *input_layer The output layer of the model that provides the inputs to the loss. + * @return Pointer to the (successfully) initialized loss structure. + */ +ailoss_t *ailoss_crossentropy(ailoss_crossentropy_t *loss, ailayer_t *input_layer); + +/** @brief Calculate the combined derivative of the given Cross-Entropy loss and the output layer for error backpropagation + * + * *Implementation of ailoss.calc_delta.* + * + * It uses the result tensor of the output layer and the target data as input and writes the result + * to the deltas tensor (ailayer.deltas) of the output layer of the model. + * + * By combining the Cross-Entropy loss with a Sigmoid or Softmax output layer, the combined deltas can be + * calculated very efficiently. The backward function of the Sigmoid / Softmax layer is not used anymore. + * + * Calculation of the deltas: + * @f[ + * \delta_{in} \leftarrow p - y + * @f] + * + * \f$ \delta_{in} \f$: Result of the delta calculation of this loss (written to ailayer.deltas of the output layer of the model)\n + * \f$ p \f$: Result of the forward pass of the output layer of the model (predicted values)\n + * \f$ y \f$: Target data / True values / Labels\n\n + * + * Used math functions: + * * ailoss_crossentropy.tensor_sub + * + * @param *self Loss to calculate the deltas for + * @param *target_data Target data / True values / Labels + */ +void ailoss_crossentropy_calc_delta(ailoss_t *self, const aitensor_t *target_data); + +/** @brief Calculate the Cross-Entropy loss on the given target data + * + * *Implementation of ailoss.calc_loss.* + * + * It uses the result tensor of the output layer and the target data as input and writes the result + * to the given result scalar. + * + * Calculation of the loss with Sigmoid output layer: + * @f[ + * result \leftarrow - \sum_i (y_i \log(p_i) + (1 - y_i) \log(1 - p_i)) + * @f] + * + * Calculation of the loss with Softmax output layer: + * @f[ + * result \leftarrow - \sum_i y_i \log(p_i) + * @f] + * + * + * \f$ result \f$: Result of the loss calculation\n + * \f$ p \f$: Result of the forward pass of the output layer of the model (predicted values)\n + * \f$ y \f$: Target data / True values / Labels\n\n + * + * Used math functions: + * * ailoss_crossentropy.tensor_sub + * * ailoss_crossentropy.crossentropy + * + * @param *self Loss to calculate the deltas for + * @param *target_data Target data / True values / Labels + * @param *result Result scalar (the data type is specified by the data type specific implementations) + */ +void ailoss_crossentropy_calc_loss(ailoss_t *self, const aitensor_t *target_data, void *result); + +/** @brief Dummy backward-function for the output layer of the model + * + * *Implementation of ailayer.backward.* +* +* The ailoss_crossentropy_calc_delta() function calculates the combined delta for a Sigmoid or Softmax layer +* with Cross-Entropy loss. Therefor the backward function of the Sigmoid / Softmax layer is not needed anymore +* and gets exchanged by this dummy function. +* +* The dummy function does nothing. +* +* @param *self The output layer of the model +*/ +void ailoss_crossentropy_dummy_backward(ailayer_t *self); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the loss specification + * + * @param *self The loss to print the specification for + * @param *print Pointer to the print function to use + */ +void ailoss_crossentropy_print_specs(const ailoss_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // CROSSENTROPY_LOSS diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailoss/ailoss_mse.c b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailoss/ailoss_mse.c new file mode 100644 index 0000000..f42d37c --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailoss/ailoss_mse.c @@ -0,0 +1,99 @@ +/** + * \file basic/base/ailoss/ailoss_mse.c + * \version 2.0alpha + * \date 20.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/ailoss/ailoss_mse.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_losstype_t ailoss_mse_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "Mean Squared Error", + .print_specs = ailoss_mse_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_losstype_t *ailoss_mse_type = &ailoss_mse_type_s; + +ailoss_t *ailoss_mse(ailoss_mse_t *loss, ailayer_t *input_layer) +{ + loss->base.loss_type = ailoss_mse_type; + + loss->base.connection_layer.input_layer = input_layer; + input_layer->output_layer = &(loss->base.connection_layer); + + loss->base.loss_configuration = loss; + + loss->base.connection_layer.deltas.dtype = loss->dtype; + loss->base.connection_layer.deltas.dim = 2; + loss->base.connection_layer.deltas.shape = input_layer->result.shape; + loss->base.connection_layer.get_result_bound = 0; + + loss->base.calc_delta = ailoss_mse_calc_delta; + loss->base.calc_loss = ailoss_mse_calc_loss; + + return &loss->base; +} + +void ailoss_mse_calc_delta(ailoss_t *self, const aitensor_t *target_data) +{ + ailoss_mse_t *loss = (ailoss_mse_t *)(self->loss_configuration); + aitensor_t *predicted_data = &(self->connection_layer.input_layer->result); + + loss->tensor_sub(predicted_data, target_data, &(self->connection_layer.deltas)); + + // ToDo: Scale by batch size. (Divide / Shift) + + return; +} + +void ailoss_mse_calc_loss(ailoss_t *self, const aitensor_t *target_data, void *result) +{ + ailoss_mse_t *loss = (ailoss_mse_t *)(self->loss_configuration); + + aitensor_t *predicted_data = &(self->connection_layer.input_layer->result); + + uint8_t temp_tensor_data[aimath_sizeof_tensor_data(target_data)]; + aitensor_t temp_tensor = { + .dim = target_data->dim, + .shape = target_data->shape, + .data = temp_tensor_data, + .dtype = target_data->dtype, + .tensor_params = self->connection_layer.deltas.tensor_params + }; + + loss->tensor_sub(predicted_data, target_data, &temp_tensor); + loss->norm_squared(&temp_tensor, result); + + // ToDo: Scale by batch size. (Divide / Shift) + + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void ailoss_mse_print_specs(const ailoss_t *self, int (*print)(const char *format, ...)) +{ + return; +} +#endif diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailoss/ailoss_mse.h b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailoss/ailoss_mse.h new file mode 100644 index 0000000..bae272e --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/ailoss/ailoss_mse.h @@ -0,0 +1,162 @@ +/** + * \file basic/base/ailoss/ailoss_mse.h + * \internal + * \date 20.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link ailoss loss \endlink implementation of the Mean Squared Error (MSE) loss + * + * This is an "abstract" data-type independent implementation. To use the loss, use one of the provided + * implementations for a specific hardware and data-type (for example from ailoss_mse_default.h) or set + * the required math functions on your own. + * + * The Mean Squared Error (MSE) loss ist best suitable for regression tasks. The loss / cost is calculated as + * @f[ + * L(y, \hat{y}) = \sum_{i=0}^{N} (y_i - \hat{y}_i)^2 + * @f] + * with the predicted values \f$ \hat{y}_i \f$ and the target values \f$ y_i \f$. + * \f$ N \f$ is the number of elements of the \f$ y \f$ tensor. + * + * To get the "mean" normalization, you have to modify the learning rate to \f$ lr = \frac {1}{o \cdot n} \cdot lr \f$ + * with the number of outputs \f$ o \f$ and the batch size \f$ n \f$. + * + * The loss can be calculated with ailoss_mse_calc_loss(). For training the deltas /errors on the target data are + * calculated with ailoss_mse_calc_delta() and written to the deltas tensor of the connection layer. + */ + +#ifndef MSE_LOSS +#define MSE_LOSS + +#include "core/aifes_core.h" + +typedef struct ailoss_mse ailoss_mse_t; /**< New data type name for code reduction. */ + +/** @brief General \link ailoss_mse.h Mean Squared Error (MSE) loss \endlink struct + * + */ +struct ailoss_mse { + ailoss_t base; /**< Inherited field members from general ailoss struct. */ + const aimath_dtype_t *dtype; /**< Main data type of the loss. */ + + /** @name Math fuctions + * @brief Required data type specific math functions + */ + ///@{ + + /** @brief Required math function: Element wise tensor subtraction + * + * Requires a math function that subtracts two tensors element wise:\n + * @f[ + * result = a - b + * @f] + */ + void (*tensor_sub)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + /** @brief Required math function: Squared sum of tensor elements + * + * Requires a math function that calculates the squared sum of all elements of a given tensor:\n + * @f[ + * result = \sum_i x_{i}^2 + * @f] + */ + void (*norm_squared)(const aitensor_t *x, void *result); + + ///@} +}; + +/** @brief Mean Squared Error loss type + * + * Defines the type of the loss (for example for type checks and debug prints). + * See aicore_losstype for more information about the loss type. + */ +extern const aicore_losstype_t *ailoss_mse_type; + +/** @brief Initialize and connect the given MSE loss + * + * This function represents the "constructor" of the abstract MSE loss. It initializes the loss structure + * and connects it to the output layer of the AIfES model.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example ailoss_mse_f32_default()). + * + * @param *loss The loss to initialize. + * @param *input_layer The output layer of the model that provides the inputs to the loss. + * @return Pointer to the (successfully) initialized loss structure. + */ +ailoss_t *ailoss_mse(ailoss_mse_t *loss, ailayer_t *input_layer); + +/** @brief Calculate the derivative of the given MSE loss for error backpropagation + * + * *Implementation of ailoss.calc_delta.* + * + * It uses the result tensor of the output layer and the target data as input and writes the result + * to the deltas tensor (ailayer.deltas) of the connection layer (ailoss.connection_layer). + * + * Calculation of the deltas: + * @f[ + * \delta_{in} \leftarrow p - y + * @f] + * + * \f$ \delta_{in} \f$: Result of the delta calculation of this loss (written to ailayer.deltas of the ailoss.connection_layer)\n + * \f$ p \f$: Result of the forward pass of the output layer of the model (predicted values)\n + * \f$ y \f$: Target data / True values / Labels\n\n + * + * Used math functions: + * * ailoss_mse.tensor_sub + * + * @param *self Loss to calculate the deltas for + * @param *target_data Target data / True values / Labels + */ +void ailoss_mse_calc_delta(ailoss_t *self, const aitensor_t *target_data); + +/** @brief Calculate the MSE loss on the given target data + * + * *Implementation of ailoss.calc_loss.* + * + * It uses the result tensor of the output layer and the target data as input and writes the result + * to the given result scalar. + * + * Calculation of the loss: + * @f[ + * result \leftarrow \sum_i (y_i - p_i)^2 + * @f] + * + * \f$ result \f$: Result of the loss calculation\n + * \f$ p \f$: Result of the forward pass of the output layer of the model (predicted values)\n + * \f$ y \f$: Target data / True values / Labels\n\n + * + * Used math functions: + * * ailoss_mse.tensor_sub + * * ailoss_mse.norm_squared + * + * @param *self Loss to calculate the deltas for + * @param *target_data Target data / True values / Labels + * @param *result Result scalar (the data type is specified by the data type specific implementations) + */ +void ailoss_mse_calc_loss(ailoss_t *self, const aitensor_t *target_data, void *result); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the loss specification + * + * @param *self The loss to print the specification for + * @param *print Pointer to the print function to use + */ +void ailoss_mse_print_specs(const ailoss_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // MSE_LOSS diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aimath/aimath_basic.c b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aimath/aimath_basic.c new file mode 100644 index 0000000..e927504 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aimath/aimath_basic.c @@ -0,0 +1,78 @@ +/** + * \file basic/base/aimath/aimath_basic.c + * \version 2.0alpha + * \date 28.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Basic math operations for aitensor_t + * \details Basic functions for aitensor_t handling, e.g. calculation of size, printing etc. + */ + +#include "basic/base/aimath/aimath_basic.h" + +void print_aitensor(const aitensor_t *tensor) +{ + tensor->dtype->print_aitensor(tensor); + return; +} + +void print_aiscalar(const void *scalar, const aimath_dtype_t *dtype) +{ + dtype->print_aiscalar(scalar, printf); + return; +} + +void aimath_transpose_vector(aitensor_t *vector) +{ + uint16_t temp; + temp = vector->shape[0]; + vector->shape[0] = vector->shape[1]; + vector->shape[1] = temp; + return; +} + +uint32_t aimath_tensor_elements(const aitensor_t *tensor) +{ + uint32_t elems = 1; + uint8_t i; + + for(i = 0; i < tensor->dim; i++) + { + elems *= tensor->shape[i]; + } + return elems; +} + +uint32_t aimath_sizeof_dtype(const aimath_dtype_t *dtype) +{ + return dtype->size; +} + +uint32_t aimath_sizeof_tensor_data(const aitensor_t *tensor) +{ + return aimath_sizeof_dtype(tensor->dtype) * aimath_tensor_elements(tensor); +} + +uint32_t aimath_sizeof_tensor_params(const aitensor_t *tensor) +{ + return tensor->dtype->tensor_params_size; +} + +uint32_t aimath_sizeof_tensor(const aitensor_t *tensor) +{ + return sizeof(aitensor_t) + aimath_sizeof_tensor_data(tensor) + aimath_sizeof_tensor_params(tensor); +} diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aimath/aimath_basic.h b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aimath/aimath_basic.h new file mode 100644 index 0000000..178d702 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aimath/aimath_basic.h @@ -0,0 +1,123 @@ +/** + * \file basic/base/aimath/aimath_basic.h + * \internal + * \date 28.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Basic data-type independent math operations + * + */ + +#ifndef AIMATH_BASIC +#define AIMATH_BASIC + +#include "core/aifes_math.h" + +/** @brief Printing a tensor to console + * + * Calls the corresponding print function of the used aimath_dtype. + * + * Example: + * \code{.c} + * uint16_t tensor_shape[2] = {2, 3}; + * float tensor_data[2*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f}; + * aitensor_t tensor = AITENSOR_2D_F32(tensor_shape, tensor_data); + * + * print_aitensor(&tensor); + * \endcode + * + * @param *tensor The tensor to print. + */ +void print_aitensor(const aitensor_t *tensor); + +/** @brief Printing a scalar to console + * + * Calls the corresponding print function of the used aimath_dtype. + * + * Example: + * \code{.c} + * aiscalar_f32_t scalar = 42.0f; + * + * print_aiscalar(&scalar, aif32); + * \endcode + * + * @param *scalar The scalar to print. + * @param *dtype The data type of the scalar + */ +void print_aiscalar(const void *scalar, const aimath_dtype_t *dtype); + +/** @brief Transposes a vector + * + * @f[ + * vector \leftarrow vector^T + * @f] + * + * Make sure that the given tensor is a vector (with shape [1 x N] or [N x 1]) + * + * @param *vector The vector to transpose (2D tensor of shape [1 x N] or [N x 1]) + */ +void aimath_transpose_vector(aitensor_t *vector); + +/** @brief Calculates the number of elements in a tensor + * + * @param *tensor The tensor to count the elements of + * @return Number of elements in tensor + */ +uint32_t aimath_tensor_elements(const aitensor_t *tensor); + +/** @brief Returns the size of one value in the given data type in bytes + * + * @param *dtype The data type to get the size of + * @return Size of one value in the given data type in bytes + */ +uint32_t aimath_sizeof_dtype(const aimath_dtype_t *dtype); + +/** @brief Returns size of the data array of the tensor in bytes + * + * The size is calculated by: + * + * \code aimath_tensor_elements(tensor) * aimath_sizeof_dtype(tensor.dtype) \endcode + * + * @param *tensor The tensor to get the data size of + * @return Size of tensor data in bytes + */ +uint32_t aimath_sizeof_tensor_data(const aitensor_t *tensor); + +/** @brief Returns size of the aitensor.tensor_params for the data type of the tensor + * + * @param *tensor The tensor to get the parameters size of + * @return Size of tensor parameters in bytes + */ +uint32_t aimath_sizeof_tensor_params(const aitensor_t *tensor); + +/** @brief Returns size of tensor in bytes + * + * The size is calculated by: + * + * \code sizeof(aitensor_t) + aimath_sizeof_tensor_data(tensor) + aimath_sizeof_tensor_params(tensor) \endcode + * + * @param *tensor The tensor to get the size of + * @return Size of tensor in bytes + */ +uint32_t aimath_sizeof_tensor(const aitensor_t *tensor); + +//void aimath_create_tensor_2d(uint16_t shape_0, uint16_t shape_1, aimath_dtype_t dtype, void *memory_ptr); + +#endif // AIMATH_BASIC diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aimath/aimath_f32.c b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aimath/aimath_f32.c new file mode 100644 index 0000000..0eaa402 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aimath/aimath_f32.c @@ -0,0 +1,91 @@ +/** + * \file basic/base/aimath/aimath_f32.c + * \version 2.0alpha + * \date 28.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Basic functions for f32 datatypes + */ + +#include "basic/base/aimath/aimath_f32.h" + +const aimath_dtype_t aif32_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "F32", +#else + .name = 0, +#endif + .size = 4, + .tensor_params_size = 0, + .print_aitensor = aimath_f32_print_aitensor, + .print_aiscalar = aimath_f32_print_aiscalar +}; + +const aimath_dtype_t *aif32 = &aif32_s; + + +void aimath_f32_print_aitensor(const aitensor_t *tensor) +{ + uint16_t i, j, k, n; + printf("F32 [\n"); + if(tensor->dim == 1) + { + for(j = 0; j < tensor->shape[0]; j++) + { + printf("%10.5f\t", ((float *) tensor->data)[j]); + } + } + else if(tensor->dim == 2) + { + for(i = 0; i < tensor->shape[0]; i++) + { + for(j = 0; j < tensor->shape[1]; j++) + { + printf("%10.5f\t", ((float *) tensor->data)[i*tensor->shape[1] + j]); + } + printf("\n"); + } + } + else if(tensor->dim == 4) + { + float (*tensor_data)[tensor->shape[0]][tensor->shape[1]][tensor->shape[2]][tensor->shape[3]] = tensor->data; + + for(n = 0; n < tensor->shape[0]; n++) + { + for(k = 0; k < tensor->shape[1]; k++) + { + for(i = 0; i < tensor->shape[2]; i++) + { + for(j = 0; j < tensor->shape[3]; j++) + { + printf("%10.5f\t", (*tensor_data)[n][k][i][j]); + } + printf("\n"); + } + printf("\n"); + } + printf("\n"); + } + } + printf("]\n"); + return; +} + +void aimath_f32_print_aiscalar(const void *scalar, int (*print)(const char *format, ...)) +{ + print("%f (F32)", *((float *) scalar)); +} diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aimath/aimath_f32.h b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aimath/aimath_f32.h new file mode 100644 index 0000000..e067fe8 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aimath/aimath_f32.h @@ -0,0 +1,134 @@ +/** + * \file basic/base/aimath/aimath_f32.h + * \internal + * \date 28.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Definition of the F32 (aif32) data-type + * + * The F32 (aif32) data-type stores data as 32 bit single precision floating point values. + * It does not require any additional parameter to define the values. + * + * **Example: Create a F32 tensor**\n + * The tensor + * @f[ + * \left( \begin{array}{rrr} 0 & 1 & 2 \\ 3 & 4 & 5 \end{array}\right) + * @f] + * can be created with + * \code{.c} + * float example_data[] = {0.0f, 1.0f, 2.0f, + * 3.0f, 4.0f, 5.0f}; + * uint16_t example_shape[] = {2, 3}; + * aitensor_t example_tensor = { + * .dtype = aif32, + * .dim = 2, + * .shape = example_shape, + * .data = example_data + * }; + * \endcode + * + * **Example: Create a F32 scalar**\n + * Either create it as a normal float value + * \code{.c} + * float scalar = 42.0f; + * \endcode + * or with the alias + * \code{.c} + * aiscalar_f32_t scalar = 42.0f; + * \endcode + * + * **Example: Print a F32 tensor to the console** + * \code{.c} + * print_aitensor(&example_tensor); + * \endcode + * + * **Example: Print a F32 scalar to the console** + * \code{.c} + * print_aiscalar(&scalar, aif32); + * \endcode + */ + +#ifndef AIMATH_F32 +#define AIMATH_F32 + +#include "core/aifes_math.h" +#include "basic/base/aimath/aimath_basic.h" + +/** @brief Initialize a 2 dimensional F32 tensor + * + * @param shape A uint16_t array of length 2 for the shape + * @param data A float array for the tensor data + */ +#define AITENSOR_2D_F32(shape, data) {aif32, 2, shape, 0, data} + +/** @brief Scalar for F32 (aif32) data-type + * + * This is only an alias for a float value for consistency.\n + * You can create a F32 scalar either as a normal float value + * \code{.c} + * float scalar = 42.0f; + * \endcode + * or with the alias + * \code{.c} + * aiscalar_f32_t scalar = 42.0f; + * \endcode + * + * You can print the scalar to the console with + * \code{.c} + * print_aiscalar(&scalar, aif32); + * \endcode + * or by using + * \code{.c} + * aimath_f32_print_aiscalar(&scalar, printf); + * \endcode + */ +typedef float aiscalar_f32_t; + +/** @brief Printing a F32 tensor to console + * + * For users the function + * \code{.c} + * print_aitensor(&tensor); + * \endcode + * is prefered. + * + * @param *tensor The tensor to print. + */ +void aimath_f32_print_aitensor(const aitensor_t *tensor); + +/** @brief Printing a F32 scalar to console + * + * For users the function + * \code{.c} + * print_aiscalar(&scalar, aif32); + * \endcode + * is prefered. + * + * @param *scalar The scalar (type: float) to print. + * @param *print The print function to use + */ +void aimath_f32_print_aiscalar(const void *scalar, int (*print)(const char *format, ...)); + +/** @brief The F32 data-type indicator + * + * Use this variable to configure some element with the \link aimath_f32.h F32 \endlink data-type, + */ +extern const aimath_dtype_t *aif32; + +#endif // AIMATH_F32 diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aiopti/aiopti_adam.c b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aiopti/aiopti_adam.c new file mode 100644 index 0000000..74c7163 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aiopti/aiopti_adam.c @@ -0,0 +1,149 @@ +/** + * \file basic/base/aiopti/aiopti_adam.c + * \version 2.0alpha + * \date 12.11.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/aiopti/aiopti_adam.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_optitype_t aiopti_adam_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "ADAM", + .print_specs = aiopti_adam_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_optitype_t *aiopti_adam_type = &aiopti_adam_type_s; + +aiopti_t *aiopti_adam(aiopti_adam_t *opti) +{ + opti->base.optimizer_type = aiopti_adam_type; + opti->base.optimizer_configuration = opti; + + opti->base.sizeof_optimem = aiopti_adam_sizeof_optimem; + opti->base.init_optimem = aiopti_adam_init_optimem; + opti->base.zero_gradients = aiopti_adam_zero_gradients; + opti->base.update_params = aiopti_adam_update_params; + + return (&opti->base); +} + +uint32_t aiopti_adam_sizeof_optimem(aiopti_t *self, const aitensor_t *params){ + uint32_t memory = 0; + memory += sizeof(aiopti_adam_momentums_t); + memory += 2 * aimath_sizeof_tensor_data(params); + return memory; +} + +void aiopti_adam_init_optimem(aiopti_t *self, const aitensor_t *params, const aitensor_t *gradients, void *optimem) +{ + aiopti_adam_t *opti = (aiopti_adam_t *)(self->optimizer_configuration); + uint32_t address_counter = 0; + aiopti_adam_momentums_t *momentums = optimem; + address_counter += sizeof(aiopti_adam_momentums_t); + + momentums->m.dtype = gradients->dtype; + momentums->m.dim = gradients->dim; + momentums->m.shape = gradients->shape; + momentums->m.tensor_params = gradients->tensor_params; + momentums->m.data = optimem + address_counter; + address_counter += aimath_sizeof_tensor_data(&momentums->m); + + momentums->v.dtype = gradients->dtype; + momentums->v.dim = gradients->dim; + momentums->v.shape = gradients->shape; + momentums->v.tensor_params = gradients->tensor_params; + momentums->v.data = optimem + address_counter; + address_counter += aimath_sizeof_tensor_data(&momentums->v); + + opti->zero_tensor(&momentums->m); + opti->zero_tensor(&momentums->v); + + return; +} + +void aiopti_adam_zero_gradients(aiopti_t *self, aitensor_t *gradients) +{ + aiopti_adam_t *opti = (aiopti_adam_t *)(self->optimizer_configuration); + + opti->zero_tensor(gradients); + + return; +} + +void aiopti_adam_update_params(aiopti_t *self, aitensor_t *params, const aitensor_t *gradients, void *optimem) +{ + aiopti_adam_t *opti = (aiopti_adam_t *)(self->optimizer_configuration); + + aiopti_adam_momentums_t *momentums = optimem; + + uint8_t temp_tensor_data[aimath_sizeof_tensor_data(gradients)]; + aitensor_t temp_tensor = { + .dim = gradients->dim, + .shape = gradients->shape, + .data = temp_tensor_data, + .dtype = gradients->dtype, + .tensor_params = gradients->tensor_params + }; + + // m = beta1 * m + (1-beta1) * g + opti->scalar_mul(opti->beta1, &momentums->m, &momentums->m); + opti->scalar_mul(opti->one_minus_beta1, gradients, &temp_tensor); + opti->tensor_add(&momentums->m, &temp_tensor, &momentums->m); + + // v = beta2 * v + (1-beta2) * g*g + opti->scalar_mul(opti->beta2, &momentums->v, &momentums->v); + opti->multiply(gradients, gradients, &temp_tensor); + opti->scalar_mul(opti->one_minus_beta2, &temp_tensor, &temp_tensor); + opti->tensor_add(&momentums->v, &temp_tensor, &momentums->v); + + // lrt = lr * sqrt(1-beta2^t) / (1-beta1)^t + // params = params - lrt * m / (sqrt(v) + eps) + opti->sqrt(&momentums->v, &temp_tensor); + opti->scalar_add(opti->eps, &temp_tensor, &temp_tensor); + opti->divide(&momentums->m, &temp_tensor, &temp_tensor); + opti->scalar_mul(opti->lrt, &temp_tensor, &temp_tensor); + + temp_tensor.tensor_params = params->tensor_params; // switch params like shift to the parameter tensor params + opti->tensor_sub(params, &temp_tensor, params); + + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void aiopti_adam_print_specs(const aiopti_t *self, int (*print)(const char *format, ...)) +{ + aiopti_adam_t *self_casted = (aiopti_adam_t *) self->optimizer_configuration; + + print("lr: "); + self->dtype->print_aiscalar(self->learning_rate, print); + print("; beta1: "); + self->dtype->print_aiscalar(self_casted->beta1, print); + print("; beta2: "); + self->dtype->print_aiscalar(self_casted->beta2, print); + print("; eps: "); + self->dtype->print_aiscalar(self_casted->eps, print); + return; +} +#endif diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aiopti/aiopti_adam.h b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aiopti/aiopti_adam.h new file mode 100644 index 0000000..fb41ae3 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aiopti/aiopti_adam.h @@ -0,0 +1,295 @@ +/** + * \file basic/base/aiopti/aiopti_adam.h + * \internal + * \date 12.11.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link aiopti optimizer \endlink implementation of the [Adam optimizer](https://arxiv.org/pdf/1412.6980.pdf) + * + * This is an "abstract" data-type independent implementation. To use the optimizer, use one of the provided + * implementations for a specific hardware and data-type (for example from aiopti_adam_default.h) or set + * the required math functions on your own. + * + * The [Adam optimizer](https://arxiv.org/pdf/1412.6980.pdf) is based on SGD and uses first-order and second-order moments for adaptive estimation. + * It uses the pre-calculated gradients to optimize the given parameters. + * For every parameter \f$ p \f$ of the parameters to optimize (trainable parameters) and the related gradient \f$ g \f$ it calculates + * @f[ + * m_t = \beta_1 \cdot m_{t-1} + (1 - \beta_1) \cdot g_t + * @f] + * @f[ + * v_t = \beta_2 \cdot v_{t-1} + (1 - \beta_2) \cdot g^2_t + * @f] + * @f[ + * p_t = p_{t-1} - lr_t \cdot \frac{m_t} {\sqrt{v_t} + \hat{\epsilon}} + * @f] + * in every optimization step with \f$ lr_t = lr \cdot \frac{\sqrt{1 - \beta^t_2}} {(1 - \beta_1)^t} \f$.\n + * \f$ lr \f$ is the learning rate that defines how big the optimization steps should be, + * and therefore how fast the training will be. + * \f$ m \f$ and \f$ v \f$ are the first- and second-order moments related to the parameter and must be stored in the optimization memory + * for every parameter. + */ + +#ifndef AIOPTI_ADAM +#define AIOPTI_ADAM + +#include "core/aifes_core.h" + +typedef struct aiopti_adam aiopti_adam_t; /**< New data type name for code reduction. */ +typedef struct aiopti_adam_momentums aiopti_adam_momentums_t; /**< New data type name for code reduction. */ + +/** @brief General \link aiopti_adam.h Adam optimizer \endlink struct +* +*/ +struct aiopti_adam { + aiopti_t base; /**< Inherited field members from general optimizer struct. */ + + /** @name Optimizer configuration + * @brief Required configuration parameters for the optimizer + * + * These fields have to be configured by the user before calling the initializer function. + */ + ///@{ + void *beta1; /**< aiscalar: Exponential decay rate for the first moment estimates \f$ \beta_1 \f$ */ + void *beta2; /**< aiscalar: Exponential decay rate for the second moment estimates \f$ \beta_2 \f$ */ + void *eps; /**< aiscalar: Small positive number for numerical stability \f$ \hat{\epsilon} \f$ (avoid dividing by 0)*/ + ///@} + + /** @name Variables for internal computation + * + * These fields are automatically configured in the initializer function. + */ + ///@{ + void *beta1t; /**< aiscalar: Auxiliary variable to calculate \f$ \beta_1^t \f$ */ + void *beta2t; /**< aiscalar: Auxiliary variable to calculate \f$ \beta_2^t \f$ */ + void *one_minus_beta1; /**< aiscalar: Auxiliary variable to calculate \f$ (1 - \beta_1) \f$ */ + void *one_minus_beta2; /**< aiscalar: Auxiliary variable to calculate \f$ (1 - \beta_2) \f$ */ + void *lrt; /**< aiscalar: Auxiliary variable to calculate \f$ lr_t = lr \cdot \frac{\sqrt{1-\beta_2^t}}{(1-\beta_1^t)} \f$ */ + ///@} + + /** @name Math functions + * @brief Required data type specific math functions + */ + ///@{ + + /** @brief Required math function: Element wise tensor multiplication + * + * Requires a math function that multiplies two tensors element wise:\n + * @f[ + * result = a \odot b + * @f] + */ + void (*multiply)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + /** @brief Required math function: Element wise tensor dividation + * + * Requires a math function that devides two tensors element wise:\n + * @f[ + * result_i = a_i \div b_i + * @f] + */ + void (*divide)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + /** @brief Required math function: Element wise tensor addition + * + * Requires a math function that adds two tensors element wise:\n + * @f[ + * result = a + b + * @f] + */ + void (*tensor_add)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + /** @brief Required math function: Element wise tensor subtraction + * + * Requires a math function that subtracts two tensors element wise:\n + * @f[ + * result = a - b + * @f] + */ + void (*tensor_sub)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + /** @brief Required math function: Multiplication of a scalar with a tensor + * + * Requires a math function that multiplies a scalar with a tensor:\n + * @f[ + * result = scalar \cdot a + * @f] + */ + void (*scalar_mul)(const void *scalar, const aitensor_t *a, aitensor_t *result); + + /** @brief Required math function: Element wise addition of a scalar to a tensor + * + * Requires a math function that adds a scalar to every element of a tensor:\n + * @f[ + * result = scalar \cdot \left( \begin{array}{c} 1 & \cdots & 1 \\ \vdots & \ddots & \vdots \\ 1 & \cdots & 1 \\ \end{array}\right) + a + * @f] + */ + void (*scalar_add)(const void *scalar, const aitensor_t *a, aitensor_t *result); + + /** @brief Required math function: Square root + * + * Requires a math function that calculates the element wise square root of a tensor:\n + * @f[ + * result_{i} = \sqrt{x_i} + * @f] + */ + void (*sqrt)(const aitensor_t *x, aitensor_t *result); + + /** @brief Required math function: Sets the elements of a tensor to zero + * + * Requires a math function that sets the elements of the given tensor to zero (or to the zero_point):\n + * @f[ + * tensor_i = 0 + * @f] + */ + void (*zero_tensor)(aitensor_t *tensor); + + ///@{ +}; + +/** @brief Adam optimizer type + * + * Defines the type of the optimizer (for example for type checks and debug prints). + * See aicore_optitype for more information about the optimizer type. + */ +extern const aicore_optitype_t *aiopti_adam_type; + +/** @brief Struct for the momentum tensors of an Adam optimizer + * + * This structure is made for easy access of the momentum tensors on the optimization memory. + * + */ +struct aiopti_adam_momentums { + aitensor_t m; /**< First momentum vector. */ + aitensor_t v; /**< Second momentum vector. */ +}; + +/** @brief Initialize the given Adam optimizer + * + * This function represents the "constructor" of the abstract Adam optimizer.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example aiopti_adam_f32_default()). + * + * @param *opti The optimizer to initialize. + * @return Pointer to the (successfully) initialized general optimizer structure (aiopti_adam.base) + */ +aiopti_t *aiopti_adam(aiopti_adam_t *opti); + +/** @brief Calculates the required memory for the optimization step + * + * *Implementation of aiopti.sizeof_optimem.* + * + * Calculates the size of the memory space that must be reserved. + * The memory is used for the first and second moment tensors and is calculated by: + * + * \code 2 * (sizeof(aitensor) + sizeof(params.data)) \endcode + * + * @param *self The optimizer + * @param *params The tensor of trainable parameters to calculate the memory for + */ +uint32_t aiopti_adam_sizeof_optimem(aiopti_t *self, const aitensor_t *params); + +/** @brief Initialization of the optimization memory buffer + * + * *Implementation of aiopti.init_optimem.* + * + * Initialize the first and second moment tensors with zeros: + * @f[ + * m_{0,i} \leftarrow 0 + * @f] + * @f[ + * v_{0,i} \leftarrow 0 + * @f] + * + * Used math functions: + * * aiopti_adam.zero_tensor + * + * @param *self The optimizer + * @param *params The tensor of trainable parameters + * @param *gradients The gradients associated to the parameters + * @param *optimem The optimization memory (containing the first and second moment) associated to the parameters + */ +void aiopti_adam_init_optimem(aiopti_t *self, const aitensor_t *params, const aitensor_t *gradients, void *optimem); + +/** @brief Set the gradients to zero + * + * *Implementation of aiopti.zero_gradients.* + * + * @f[ + * g_{i} \leftarrow 0 + * @f] + * + * Used math functions: + * * aiopti_adam.zero_tensor + * + * @param *self The optimizer + * @param *gradients The gradients to set to zero + */ +void aiopti_adam_zero_gradients(aiopti_t *self, aitensor_t *gradients); + +/** @brief Update the given parameter tensor with respect to the gradients + * + * *Implementation of aiopti.update_params.* + * + * Calculate and update the values of the trainable parameters (perform one update step): + * @f[ + * m_t \leftarrow \beta_1 \cdot m_{t-1} + (1 - \beta_1) \cdot g_t + * @f] + * @f[ + * v_t \leftarrow \beta_2 \cdot v_{t-1} + (1 - \beta_2) \cdot g^2_t + * @f] + * @f[ + * p_t \leftarrow p_{t-1} - lr_t \cdot \frac{m_t} {\sqrt{v_t} + \hat{\epsilon}} + * @f] + * + * \f$ m \f$: First moment estimates\n + * \f$ v \f$: Second moment estimates\n + * \f$ p \f$: Tensor of trainable parameters to update (params)\n + * \f$ g \f$: Gradients\n + * \f$ lr \f$: Learning rate / Optimization step size\n + * \f$ \beta_1 \f$: Exponential decay rate for the first moment estimates\n + * \f$ \beta_2 \f$: Exponential decay rate for the second moment estimates\n + * \f$ \hat{\epsilon} \f$: Small positive number for numerical stability\n\n + * + * Used math functions: + * * aiopti_adam.scalar_mul + * * aiopti_adam.tensor_add + * * aiopti_adam.multiply + * * aiopti_adam.sqrt + * * aiopti_adam.scalar_add + * * aiopti_adam.divide + * * aiopti_adam.tensor_sub + * + * @param *self The optimizer + * @param *params The tensor of trainable parameters \f$ p \f$ to update + * @param *gradients The gradients \f$ g \f$ associated to the parameters + * @param *optimem The buffer to store the first and second momentums \f$ m \f$ and \f$ v \f$ + */ +void aiopti_adam_update_params(aiopti_t *self, aitensor_t *params, const aitensor_t *gradients, void *optimem); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the optimizer specification + * + * @param *self The optimizer to print the specification for + * @param *print Pointer to the print function to use + */ +void aiopti_adam_print_specs(const aiopti_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // AIOPTI_ADAM + diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aiopti/aiopti_sgd.c b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aiopti/aiopti_sgd.c new file mode 100644 index 0000000..42854fb --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aiopti/aiopti_sgd.c @@ -0,0 +1,158 @@ +/** + * \file basic/base/aiopti/aiopti_sgd.c + * \version 2.0alpha + * \date 20.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/aiopti/aiopti_sgd.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_optitype_t aiopti_sgd_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "SGD", + .print_specs = aiopti_sgd_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_optitype_t *aiopti_sgd_type = &aiopti_sgd_type_s; + +aiopti_t *aiopti_sgd(aiopti_sgd_t *opti) +{ + opti->base.optimizer_type = aiopti_sgd_type; + opti->base.optimizer_configuration = opti; + + opti->base.begin_step = 0; + opti->base.end_step = 0; + opti->base.zero_gradients = aiopti_sgd_zero_gradients; + + // Need to be set in the inherited implementation + opti->base.update_params = 0; + opti->base.sizeof_optimem = 0; + opti->base.init_optimem = 0; + + return &opti->base; +} + + +uint32_t aiopti_sgd_sizeof_optimem_with_momentum(aiopti_t *self, const aitensor_t *params){ + uint32_t memory = 0; + memory += sizeof(aitensor_t); + memory += aimath_sizeof_tensor_data(params); + return memory; +} + +uint32_t aiopti_sgd_sizeof_optimem_without_momentum(aiopti_t *self, const aitensor_t *params){ + return 0; +} + +void aiopti_sgd_init_optimem_with_momentum(aiopti_t *self, const aitensor_t *params, const aitensor_t *gradients, void *optimem) +{ + aiopti_sgd_t *opti = (aiopti_sgd_t *)(self->optimizer_configuration); + uint32_t address_counter = 0; + aitensor_t *v = (aitensor_t *) optimem; + address_counter += sizeof(aitensor_t); + + v->dtype = gradients->dtype; + v->dim = gradients->dim; + v->shape = gradients->shape; + v->tensor_params = gradients->tensor_params; + v->data = optimem + address_counter; + address_counter += aimath_sizeof_tensor_data(v); + + opti->zero_tensor(v); + + return; +} + +void aiopti_sgd_init_optimem_without_momentum(aiopti_t *self, const aitensor_t *params, const aitensor_t *gradients, void *optimem) +{ + return; +} + +void aiopti_sgd_zero_gradients(aiopti_t *self, aitensor_t *gradients) +{ + aiopti_sgd_t *opti = (aiopti_sgd_t *)(self->optimizer_configuration); + + opti->zero_tensor(gradients); + + return; +} + +void aiopti_sgd_update_params_with_momentum(aiopti_t *self, aitensor_t *params, const aitensor_t *gradients, void *optimem) +{ + aiopti_sgd_t *opti = (aiopti_sgd_t *)(self->optimizer_configuration); + + aitensor_t *v = (aitensor_t *) optimem; + + uint8_t temp_tensor_data[aimath_sizeof_tensor_data(params)]; + aitensor_t temp_tensor = { + .dim = gradients->dim, + .shape = gradients->shape, + .data = temp_tensor_data, + .dtype = gradients->dtype, + .tensor_params = gradients->tensor_params + }; + + // v = momentum * v + gradients + opti->scalar_mul(opti->momentum, v, &temp_tensor); + opti->tensor_add(&temp_tensor, gradients, v); + + // p = p - lr * v + opti->scalar_mul(opti->base.learning_rate, v, &temp_tensor); + opti->tensor_sub(params, &temp_tensor, params); + + return; +} + +void aiopti_sgd_update_params_without_momentum(aiopti_t *self, aitensor_t *params, const aitensor_t *gradients, void *optimem) +{ + aiopti_sgd_t *opti = (aiopti_sgd_t *)(self->optimizer_configuration); + + uint8_t temp_tensor_data[aimath_sizeof_tensor_data(params)]; + aitensor_t temp_tensor = { + .dim = gradients->dim, + .shape = gradients->shape, + .data = temp_tensor_data, + .dtype = gradients->dtype, + .tensor_params = gradients->tensor_params + }; + + // p = p - lr * g + opti->scalar_mul(opti->base.learning_rate, gradients, &temp_tensor); + opti->tensor_sub(params, &temp_tensor, params); + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void aiopti_sgd_print_specs(const aiopti_t *self, int (*print)(const char *format, ...)) +{ + aiopti_sgd_t *self_casted = (aiopti_sgd_t *) self->optimizer_configuration; + + print("lr: "); + self->dtype->print_aiscalar(self->learning_rate, print); + + print("; momentum: "); + self->dtype->print_aiscalar(self_casted->momentum, print); + return; +} +#endif diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aiopti/aiopti_sgd.h b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aiopti/aiopti_sgd.h new file mode 100644 index 0000000..f518278 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/base/aiopti/aiopti_sgd.h @@ -0,0 +1,272 @@ +/** + * \file basic/base/aiopti/aiopti_sgd.h + * \internal + * \date 20.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link aiopti optimizer \endlink implementation of the Stochastic Gradient Descent (with momentum) optimizer + * + * This is an "abstract" data-type independent implementation. To use the optimizer, use one of the provided + * implementations for a specific hardware and data-type (for example from aiopti_sgd_default.h) or set + * the required math functions on your own. + * + * The Stochastic Gradient Descent (SGD) optimizer is the most basic optimizer in backpropagation. + * It uses the pre-calculated gradients to optimize the given parameters. In addition, a momentum term can be configured.\n + * For every parameter \f$ p \f$ of the parameters to optimize (trainable parameters) and the related gradient \f$ g \f$ it calculates + * @f[ + * p_t = p_{t-1} - lr \cdot g_t + * @f] + * if the momentum \f$ \mu = 0 \f$ and + * @f[ + * v_t = \mu \cdot v_{t-1} + g_t + * @f] + * @f[ + * p_t = p_{t-1} - lr \cdot v_t + * @f] + * if the momentum \f$ \mu \neq 0 \f$ in every optimization step.\n + * \f$ lr \f$ is the learning rate that defines how big the optimization steps should be, + * and therefore how fast the training will be. + * \f$ v \f$ is the momentum term or velocity related to the parameter and must be stored in the optimization memory + * for every parameter when momentum is set. + */ + +#ifndef AIOPTI_SGD +#define AIOPTI_SGD + +#include "core/aifes_core.h" + +typedef struct aiopti_sgd aiopti_sgd_t; /**< New data type name for code reduction. */ + +/** @brief General \link aiopti_sgd.h Stochastic Gradient Descent (SGD) optimizer \endlink struct +* +*/ +struct aiopti_sgd { + aiopti_t base; /**< Inherited field members from general optimizer struct. */ + + /** @name Optimizer configuration + * @brief Required configuration parameters for the optimizer + * + * These fields have to be configured by the user before calling the initializer function. + */ + ///@{ + void *momentum; /**< aiscalar: Momentum(set to null to save optimization memory) */ + ///@} + + /** @name Math functions + * @brief Required data type specific math functions + */ + ///@{ + + /** @brief Required math function: Element wise tensor subtraction + * + * Requires a math function that subtracts two tensors element wise:\n + * @f[ + * result = a - b + * @f] + */ + void (*tensor_sub)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + /** @brief Required math function: Element wise tensor addition + * + * Requires a math function that adds two tensors element wise:\n + * @f[ + * result = a + b + * @f] + */ + void (*tensor_add)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + /** @brief Required math function: Multiplication of a scalar with a tensor + * + * Requires a math function that multiplies a scalar with a tensor:\n + * @f[ + * result = scalar \cdot a + * @f] + */ + void (*scalar_mul)(const void *scalar, const aitensor_t *a, aitensor_t *result); + + /** @brief Required math function: Sets the elements of a tensor to zero + * + * Requires a math function that sets the elements of the given tensor to zero (or to the zero_point):\n + * @f[ + * tensor_i = 0 + * @f] + */ + void (*zero_tensor)(aitensor_t *tensor); + + ///@} +}; + +/** @brief SGD optimizer type + * + * Defines the type of the optimizer (for example for type checks and debug prints). + * See aicore_optitype for more information about the optimizer type. + */ +extern const aicore_optitype_t *aiopti_sgd_type; + + +/** @brief Initialize the given SGD optimizer + * + * This function represents the "constructor" of the abstract SGD optimizer.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example aiopti_sgd_f32_default()). + * + * @param *opti The optimizer to initialize. + * @return Pointer to the (successfully) initialized general optimizer structure (aiopti_sgd.base) + */ +aiopti_t *aiopti_sgd(aiopti_sgd_t *opti); + +/** @brief Calculates the required memory for the optimization step when the momentum is not zero + * + * *Implementation of aiopti.sizeof_optimem.* + * + * Calculates the size of the memory space that must be reserved. + * The memory is used for the velocity tensor (momentum term) and is calculated by: + * + * \code sizeof(aitensor) + sizeof(params.data) \endcode + * + * @param *self The optimizer + * @param *params The tensor of trainable parameters to calculate the memory for + */ +uint32_t aiopti_sgd_sizeof_optimem_with_momentum(aiopti_t *self, const aitensor_t *params); + +/** @brief Calculates the required memory for the optimization step when the momentum is zero + * + * *Implementation of aiopti.sizeof_optimem.* + * + * Calculates the size of the memory space that must be reserved. + * The required memory is zero because no velocity term is needed. + * + * @param *self The optimizer + * @param *params The tensor of trainable parameters to calculae the memory for + */ +uint32_t aiopti_sgd_sizeof_optimem_without_momentum(aiopti_t *self, const aitensor_t *params); + +/** @brief Initialization of the optimization memory buffer when the momentum is not zero + * + * *Implementation of aiopti.init_optimem.* + * + * Initialize the velocity tensor with zeros: + * @f[ + * v_{0,i} \leftarrow 0 + * @f] + * + * Used math functions: + * * aiopti_sgd.zero_tensor + * + * @param *self The optimizer + * @param *params The tensor of trainable parameters + * @param *gradients The gradients associated to the parameters + * @param *optimem The optimization memory (containing the velocities) associated to the parameters + */ +void aiopti_sgd_init_optimem_with_momentum(aiopti_t *self, const aitensor_t *params, const aitensor_t *gradients, void *optimem); + +/** @brief Initialization of the optimization memory buffer when the momentum is zero + * + * *Implementation of aiopti.init_optimem.* + * + * Does nothing because no optimization memory is needed in this case. + * + * @param *self The optimizer + * @param *params The tensor of trainable parameters + * @param *gradients The gradients associated to the parameters + * @param *optimem The optimization memory associated to the parameters + */ +void aiopti_sgd_init_optimem_without_momentum(aiopti_t *self, const aitensor_t *params, const aitensor_t *gradients, void *optimem); + +/** @brief Set the gradients to zero + * + * *Implementation of aiopti.zero_gradients.* + * + * @f[ + * g_{i} \leftarrow 0 + * @f] + * + * Used math functions: + * * aiopti_sgd.zero_tensor + * + * @param *self The optimizer + * @param *gradients The gradients to set to zero + */ +void aiopti_sgd_zero_gradients(aiopti_t *self, aitensor_t *gradients); + +/** @brief Update the given parameter tensor with respect to the gradients when the momentum is not zero + * + * *Implementation of aiopti.update_params.* + * + * Calculate and update the values of the trainable parameters (perform one update step): + * @f[ + * v_t \leftarrow \mu \cdot v_{t-1} + g_t + * @f] + * @f[ + * p_t \leftarrow p_{t-1} - lr \cdot v_t + * @f] + * + * \f$ v \f$: Velocity tensor (momentum term), stored in the optimem\n + * \f$ p \f$: Tensor of trainable parameters to update (params)\n + * \f$ g \f$: Gradients\n + * \f$ lr \f$: Learning rate / Optimization step size\n + * \f$ \mu \f$: Momentum\n\n + * + * Used math functions: + * * aiopti_sgd.scalar_mul + * * aiopti_sgd.tensor_add + * * aiopti_sgd.tensor_sub + * + * @param *self The optimizer + * @param *params The tensor of trainable parameters \f$ p \f$ to update + * @param *gradients The gradients \f$ g \f$ associated to the parameters + * @param *optimem The buffer to store the velocity \f$ v \f$ + */ +void aiopti_sgd_update_params_with_momentum(aiopti_t *self, aitensor_t *params, const aitensor_t *gradients, void *optimem); + +/** @brief Update the given parameter tensor with respect to the gradients when the momentum is zero + * + * *Implementation of aiopti.update_params.* + * + * Calculate and update the values of the trainable parameters (perform one update step): + * @f[ + * p_t \leftarrow p_{t-1} - lr \cdot g_t + * @f] + * + * Used math functions: + * * aiopti_sgd.scalar_mul + * * aiopti_sgd.tensor_sub + * + * \f$ p \f$: Tensor of trainable parameters to update (params)\n + * \f$ g \f$: Gradients\n + * \f$ lr \f$: Learning rate / Optimization step size\n\n + * + * @param *self The optimizer + * @param *params The tensor of trainable parameters \f$ p \f$ to update + * @param *gradients The gradients \f$ g \f$ associated to the parameters + * @param *optimem Not required because no velocity is stored + */ +void aiopti_sgd_update_params_without_momentum(aiopti_t *self, aitensor_t *params, const aitensor_t *gradients, void *optimem); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the optimizer specification + * + * @param *self The optimizer to print the specification for + * @param *print Pointer to the print function to use + */ +void aiopti_sgd_print_specs(const aiopti_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // AIOPTI_SGD + diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/cmsis/ailayer/ailayer_dense_cmsis.c b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/cmsis/ailayer/ailayer_dense_cmsis.c new file mode 100644 index 0000000..a978db1 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/cmsis/ailayer/ailayer_dense_cmsis.c @@ -0,0 +1,49 @@ +/** + * \file basic/cmsis/ailayer/ailayer_dense_cmsis.c + * \version 2.0alpha + * \date 15.03.2021 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/ailayer/ailayer_dense_default.h" +#include "../../../aifes.h" + +#if __arm__ +#ifdef AIFES_WITH_CMSIS + +#include "basic/cmsis/ailayer/ailayer_dense_cmsis.h" +#include "basic/cmsis/aimath/aimath_f32_cmsis.h" + + +ailayer_t *ailayer_dense_f32_cmsis(ailayer_dense_t *layer, ailayer_t *input_layer) +{ + layer->result_dtype = aif32; + layer->weights_dtype = aif32; + layer->bias_dtype = aif32; + + layer->linear = aimath_f32_cmsis_linear; + layer->mat_mul = aimath_f32_cmsis_mat_mul; + layer->tensor_add = aimath_f32_default_tensor_add; + + return ailayer_dense(layer, input_layer); +} + +#endif // AIFES_WITH_CMSIS +#endif //__arm__ diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/cmsis/ailayer/ailayer_dense_cmsis.h b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/cmsis/ailayer/ailayer_dense_cmsis.h new file mode 100644 index 0000000..4945e23 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/cmsis/ailayer/ailayer_dense_cmsis.h @@ -0,0 +1,77 @@ +/** + * \file basic/cmsis/ailayer/ailayer_dense_cmsis.h + * \internal + * \date 15.03.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief [Arm CMSIS](https://developer.arm.com/tools-and-software/embedded/cmsis) implementation of the \link ailayer_dense.h Dense layer \endlink for Arm Cortex processors. + * + * Arm CMSIS implementations of the Dense layer in \link aimath_f32.h F32 \endlink data-type. + * These implementations are specifically designed for the Arm Cortex processors and take advantage of SIMD instructions. + * For more information about the Dense layer refer to ailayer_dense.h. + */ + +#ifndef AILAYER_DENSE_CMSIS +#define AILAYER_DENSE_CMSIS + +#include "../../../aifes.h" + +#if __arm__ +#ifdef AIFES_WITH_CMSIS + +#include "basic/base/ailayer/ailayer_dense.h" + +#include "basic/cmsis/aimath/aimath_f32_cmsis.h" + +/** @brief Initializes and connect a Dense layer with the \link aimath_f32.h F32 \endlink CMSIS implementation + * + * Example: Create the layer structure with pretrained weights:\n + * \code{.c} + * // Use constant data only for inference. For training remove the const qualifier!! + * const float weights_data_dense[] = {-10.1164f, -8.4212f, 5.4396f, 7.297f, -7.6482f, -9.0155f}; + * const float bias_data_dense[] = {-2.9653f, 2.3677f, -1.5968f}; + * ailayer_dense_f32_t dense_layer = { + * .neurons = 3, + * .weights.data = (float *) weights_data_dense, + * .bias.data = (float *) bias_data_dense + * }; + * \endcode + * + * Example: Create the layer structure for training:\n + * \code{.c} + * ailayer_dense_f32_t dense_layer = { + * .neurons = 3 + * }; + * \endcode + * + * Example: Initialize and connect the layer:\n + * \code{.c} + * x = ailayer_dense_f32_cmsis(&dense_layer, x); + * \endcode + * + * @param *layer The layer structure to initialize. + * @param *input_layer The prior layer. + * @return The (successfully) initialized layer structure. + */ +ailayer_t *ailayer_dense_f32_cmsis(ailayer_dense_t *layer, ailayer_t *input_layer); + +#endif // AILAYER_DENSE_CMSIS + +#endif // AIFES_WITH_CMSIS +#endif //__arm__ diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/cmsis/aimath/aimath_f32_cmsis.c b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/cmsis/aimath/aimath_f32_cmsis.c new file mode 100644 index 0000000..5f8a2bc --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/cmsis/aimath/aimath_f32_cmsis.c @@ -0,0 +1,104 @@ +/** + * \file basic/cmsis/aimath/aimath_f32_cmsis.c + * \version 2.0alpha + * \date 16.03.2021 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + + +#include "basic/cmsis/aimath/aimath_f32_cmsis.h" +#include "../../../aifes.h" + +#if __arm__ +#ifdef AIFES_WITH_CMSIS +#include "arm_math.h" + +/** +* Math CMSIS Matrix Multiplication and boradtcast Ass +* +* Matrixmultiplication using the CMSIS DSP Library and broadcast add using for loop +* +*/ +void aimath_f32_cmsis_linear(const aitensor_t *a, const aitensor_t *b, const aitensor_t *c, aitensor_t *result) +{ + +#ifdef SHAPE_CHECK + if(a->shape[1] != b->shape[0]) + { + LOG_E("MatMul input shapes doesn't match.\n"); + return; + } + if(a->shape[0] != result->shape[0] || b->shape[1] != result->shape[1]) + { + LOG_E("MatMul output shape doesn't match.\n"); + return; + } +#endif + + uint16_t i, j; + + float *c_data = c != 0 ? (float *) c->data : 0; + float *result_data = (float *) result->data; + + + aimath_f32_cmsis_mat_mul(a, b, result); + + + for(i = 0; i < result->shape[0]; i++) + { + for(j = 0; j < result->shape[1]; j++) + { + result_data[i*result->shape[1] + j] += c_data[j]; + } + } + + + + return; +} + + +/** +* Math CMSIS Matrix Multiplication +* +* Matrixmultiplication using the CMSIS DSP Library +* +* \ref https://arm-software.github.io/CMSIS_5/DSP/html/group__MatrixMult.html +*/ +void aimath_f32_cmsis_mat_mul(const aitensor_t *a, const aitensor_t *b, aitensor_t *result) +{ + float *a_data = (float *) a->data; + float *b_data = (float *) b->data; + float *result_data = (float *) result->data; + + + arm_matrix_instance_f32 a_mat; /* Matrix a Instance */ + arm_matrix_instance_f32 b_mat; /* Matrix b Instance */ + arm_matrix_instance_f32 result_mat; /* Matrix result Instance */ + + arm_mat_init_f32(&a_mat, a->shape[0], a->shape[1], a_data); + arm_mat_init_f32(&b_mat, b->shape[0], b->shape[1], b_data); + arm_mat_init_f32(&result_mat, result->shape[0], result->shape[1], result_data); + + + arm_mat_mult_f32(&a_mat, &b_mat, &result_mat); +} +#endif // AIFES_WITH_CMSIS +#endif // __arm__ \ No newline at end of file diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/cmsis/aimath/aimath_f32_cmsis.h b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/cmsis/aimath/aimath_f32_cmsis.h new file mode 100644 index 0000000..199bab1 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/cmsis/aimath/aimath_f32_cmsis.h @@ -0,0 +1,130 @@ +/** + * \file basic/cmsis/aimath/aimath_f32_cmsis.h + * \internal + * \date 15.03.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ +#ifndef AIMATH_F32_CMSIS +#define AIMATH_F32_CMSIS + + +#include "basic/base/aimath/aimath_f32.h" +#include "../../../aifes.h" + +#if __arm__ +#ifdef AIFES_WITH_CMSIS + + +/** @brief Performs a matrix multiplication of f32 tensors a and b and adds a 1D tensor c to each row, using the ARM CMSIS DSP + * + * @details The addition of the horizontal vector c is performed via broadcast, i.e. elementwise in each column + * Mathematically this broadcast is equal to multiplying c with an vertical vector of ones (with the same number of elements as c) + * + * @details + * and adding the result to a * b + * @f[ + * result = a \cdot b + \left( \begin{array}{r} + 1 \\ + 1 \\ + \vdots \\ + 1 \\ + \end{array}\right) \cdot c + * @f] + * + * Example: + * @f[ + * a = \left( \begin{array}{rrr} + 1 & 2 & 3 \\ + 4 & 5 & 6 \\ + 7 & 8 & 9 + \end{array}\right) + * @f] + * + * @f[ + * b = \left( \begin{array}{rr} + 1 & 0 \\ + 0 & 1 \\ + 0 & 0 + \end{array}\right) + * @f] + * + * @f[ + * c = \left( \begin{array}{rr} + 2 & 5 + \end{array}\right) + * @f] + * + * @f[ + * result = a \cdot b + \left( \begin{array}{r} + 1 \\ + 1 \\ + 1 \\ + \end{array}\right) \cdot c + * @f] + * + * @f[ + * = \left( \begin{array}{rr} + 1 & 2 \\ + 4 & 5 \\ + 7 & 8 + \end{array}\right) + + + \left( \begin{array}{rr} + 2 & 5 \\ + 2 & 5 \\ + 2 & 5 + \end{array}\right) + * @f] + * + * @details + * + * @f[ + * = \left( \begin{array}{rr} + 3 & 7 \\ + 6 & 10 \\ + 9 & 13 + \end{array}\right) + * @f] + * + * @param *a f32 tensor a + * @param *b f32 tensor b + * @param *c Tensor c, 1 row and as many columns as the result + * @param *result Result f32 tensor + */ +void aimath_f32_cmsis_linear(const aitensor_t *a, const aitensor_t *b, const aitensor_t *c, aitensor_t *result); + +/** @brief Performs a matrix multiplication of f32 tensors a and b, using the ARM CMSIS DSP + * + * @details + * @f[ + * result = a \cdot b + * @f] + * + * @param *a f32 tensor a + * @param *b f32 tensor b + * @param *result Result f32 tensor of the multiplication + */ +void aimath_f32_cmsis_mat_mul(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + +#endif // AIFES_WITH_CMSIS +#endif //__arm__ +#endif // AIMATH_F32_CMSIS diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_dense_default.c b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_dense_default.c new file mode 100644 index 0000000..96d6e85 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_dense_default.c @@ -0,0 +1,39 @@ +/** + * \file basic/default/ailayer/ailayer_dense_default.c + * \version 2.0alpha + * \date 28.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief See ailayer_dense_default.h for documentation. + * \details + */ + +#include "basic/default/ailayer/ailayer_dense_default.h" + + +ailayer_t *ailayer_dense_f32_default(ailayer_dense_f32_t *layer, ailayer_t *input_layer) +{ + layer->result_dtype = aif32; + layer->weights_dtype = aif32; + layer->bias_dtype = aif32; + + layer->linear = aimath_f32_default_linear; + layer->mat_mul = aimath_f32_default_mat_mul; + layer->tensor_add = aimath_f32_default_tensor_add; + + return ailayer_dense(layer, input_layer); +} diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_dense_default.h b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_dense_default.h new file mode 100644 index 0000000..26c31f2 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_dense_default.h @@ -0,0 +1,69 @@ +/** + * \file basic/default/ailayer/ailayer_dense_default.h + * \internal + * \date 28.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link ailayer_dense.h Dense layer \endlink + * + * Hardware independent implementations of the Dense layer in \link aimath_f32.h F32 \endlink data-type. + * For more information about the Dense layer refer to ailayer_dense.h. + */ + +#ifndef AILAYER_DENSE_DEFAULT +#define AILAYER_DENSE_DEFAULT + +#include "basic/base/ailayer/ailayer_dense.h" +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct ailayer_dense ailayer_dense_f32_t; + +/** @brief Initializes and connect a \link ailayer_dense.h Dense layer \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * Example: Create the layer structure with pretrained weights:\n + * \code{.c} + * // Use constant data only for inference. For training remove the const qualifier!! + * const float weights_data_dense[] = {-10.1164f, -8.4212f, 5.4396f, 7.297f, -7.6482f, -9.0155f}; + * const float bias_data_dense[] = {-2.9653f, 2.3677f, -1.5968f}; + * ailayer_dense_f32_t dense_layer = { + * .neurons = 3, + * .weights.data = (float *) weights_data_dense, + * .bias.data = (float *) bias_data_dense + * }; + * \endcode + * + * Example: Create the layer structure for training:\n + * \code{.c} + * ailayer_dense_f32_t dense_layer = { + * .neurons = 3 + * }; + * \endcode + * + * Example: Initialize and connect the layer:\n + * \code{.c} + * x = ailayer_dense_f32_default(&dense_layer, x); + * \endcode + * + * @param *layer The layer structure to initialize. + * @param *input_layer The prior layer. + * @return The (successfully) initialized layer structure. + */ +ailayer_t *ailayer_dense_f32_default(ailayer_dense_f32_t *layer, ailayer_t *input_layer); + +#endif // AILAYER_DENSE_DEFAULT diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_elu_default.c b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_elu_default.c new file mode 100644 index 0000000..d6b65ef --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_elu_default.c @@ -0,0 +1,40 @@ +/** + * \file basic/default/ailayer/ailayer_elu_default.c + * \version 2.0alpha + * \date 16.03.2021 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/ailayer/ailayer_elu_default.h" + +ailayer_t *ailayer_elu_f32_default(ailayer_elu_f32_t *layer, ailayer_t *input_layer) +{ + layer->base.dtype = aif32; + layer->base.alpha = &(layer->alpha); + + //forward + layer->base.elu = aimath_f32_default_elu; + + // backward + layer->base.d_elu = aimath_f32_default_d_elu; + layer->base.multiply = aimath_f32_default_multiply; + + return ailayer_elu(&layer->base, input_layer); +} diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_elu_default.h b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_elu_default.h new file mode 100644 index 0000000..e1548c4 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_elu_default.h @@ -0,0 +1,66 @@ +/** + * \file basic/default/ailayer/ailayer_elu_default.h + * \internal + * \date 16.03.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link ailayer_elu.h ELU layer \endlink + * + * Hardware independent implementations of the ELU layer in \link aimath_f32.h F32 \endlink data-type. + * For more information about the ELU layer refer to ailayer_elu.h. + */ +#ifndef AILAYER_ELU_DEFAULT +#define AILAYER_ELU_DEFAULT + +#include "basic/base/ailayer/ailayer_elu.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct ailayer_elu_f32 ailayer_elu_f32_t; + +/** @brief Data-type specific ELU layer struct for \link aimath_f32.h F32 \endlink + * + * Adds a data field for the constant alpha in \link aimath_f32.h F32 \endlink to the base implementation. + */ +struct ailayer_elu_f32 { + ailayer_elu_t base; /**< Inherited field members from general ailayer_elu struct. */ + aiscalar_f32_t alpha; /**< Data-type specific parameter used to calculate ELU function for input values < 0. */ +}; + +/** @brief Initializes and connect an \link ailayer_elu.h ELU layer \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * Example: Create the layer structure:\n + * \code{.c} + * ailayer_elu_f32_t elu_layer = { + * .alpha = 1.0f + * }; + * \endcode + * + * Example: Initialize and connect the layer:\n + * \code{.c} + * x = ailayer_elu_f32_default(&elu_layer, x); + * \endcode + * + * @param *layer The layer structure to initialize. + * @param *input_layer The prior layer. + * @return The (successfully) initialized layer structure. + */ +ailayer_t *ailayer_elu_f32_default(ailayer_elu_f32_t *layer, ailayer_t *input_layer); + +#endif // AILAYER_ELU_DEFAULT diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_input_default.c b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_input_default.c new file mode 100644 index 0000000..5d77f44 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_input_default.c @@ -0,0 +1,32 @@ +/** + * \file basic/default/ailayer/ailayer_input_default.c + * \version 2.0alpha + * \date 12.11.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/ailayer/ailayer_input_default.h" + +ailayer_t *ailayer_input_f32_default(ailayer_input_f32_t *layer) +{ + layer->dtype = aif32; + + return ailayer_input(layer); +} diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_input_default.h b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_input_default.h new file mode 100644 index 0000000..669fa47 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_input_default.h @@ -0,0 +1,59 @@ +/** + * \file basic/default/ailayer/ailayer_input_default.h + * \internal + * \date 19.11.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link ailayer_input.h Input layer \endlink + * + * Hardware independent implementations of the Input layer in \link aimath_f32.h F32 \endlink data-type. + * For more information about the Input layer refer to ailayer_input.h. + */ + +#ifndef AILAYER_INPUT_DEFAULT +#define AILAYER_INPUT_DEFAULT + +#include "basic/base/ailayer/ailayer_input.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct ailayer_input ailayer_input_f32_t; + +/** @brief Initializes and connect an \link ailayer_input.h Input layer \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * Example: Create the layer structure:\n + * \code{.c} + * uint16_t input_layer_shape[] = {1, 2}; + * ailayer_input_f32_t input_layer = { + * .input_dim = 2, + * .input_shape = input_layer_shape + * }; + * \endcode + * + * Example: Initialize and connect the layer:\n + * \code{.c} + * x = ailayer_input_f32_default(&input_layer); + * \endcode + * + * @param *layer The layer structure to initialize. + * @return The (successfully) initialized layer structure. + */ +ailayer_t *ailayer_input_f32_default(ailayer_input_f32_t *layer); + +#endif // AILAYER_INPUT_DEFAULT diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_leaky_relu_default.c b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_leaky_relu_default.c new file mode 100644 index 0000000..4219579 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_leaky_relu_default.c @@ -0,0 +1,40 @@ +/** + * \file basic/default/ailayer/ailayer_leaky_relu_default.c + * \version 2.0alpha + * \date 16.03.2021 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/ailayer/ailayer_leaky_relu_default.h" + +ailayer_t *ailayer_leaky_relu_f32_default(ailayer_leaky_relu_f32_t *layer, ailayer_t *input_layer) +{ + layer->base.dtype = aif32; + layer->base.alpha = &(layer->alpha); + + //forward + layer->base.leaky_relu = aimath_f32_default_leaky_relu; + + // backward + layer->base.d_leaky_relu = aimath_f32_default_d_leaky_relu; + layer->base.multiply = aimath_f32_default_multiply; + + return ailayer_leaky_relu(&layer->base, input_layer); +} diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_leaky_relu_default.h b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_leaky_relu_default.h new file mode 100644 index 0000000..e0199bd --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_leaky_relu_default.h @@ -0,0 +1,68 @@ + +/** + * \file basic/default/ailayer/ailayer_leaky_relu_default.h + * \internal + * \date 16.03.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link ailayer_leaky_relu.h Leaky ReLU layer \endlink + * + * Hardware independent implementations of the Leaky ReLU layer in \link aimath_f32.h F32 \endlink data-type. + * For more information about the Leaky ReLU layer refer to ailayer_leaky_relu.h. + */ + +#ifndef AILAYER_LEAKY_RELU_DEFAULT +#define AILAYER_LEAKY_RELU_DEFAULT + +#include "basic/base/ailayer/ailayer_leaky_relu.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct ailayer_leaky_relu_f32 ailayer_leaky_relu_f32_t; + +/** @brief Data-type specific Leaky ReLU layer struct for \link aimath_f32.h F32 \endlink + * + * Adds a data field for the constant alpha in \link aimath_f32.h F32 \endlink to the base implementation. + */ +struct ailayer_leaky_relu_f32 { + ailayer_leaky_relu_t base; /**< Inherited field members from general ailayer_leaky_relu struct. */ + aiscalar_f32_t alpha; /**< Data type-specific parameter used to calculate Leaky ReLU function for input values < 0. */ +}; + +/** @brief Initializes and connect a \link ailayer_leaky_relu.h Leaky ReLU layer \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * Example: Create the layer structure:\n + * \code{.c} + * ailayer_leaky_relu_f32_t leaky_relu_layer = { + * .alpha = 0.01f + * }; + * \endcode + * + * Example: Initialize and connect the layer:\n + * \code{.c} + * x = ailayer_leaky_relu_f32_default(&leaky_relu_layer, x); + * \endcode + * + * @param *layer the layer structure to be initialized + * @param *input_layer the prior layer that provides the input to this Leaky ReLU layer + * @return the initialized Leaky ReLU layer structure + */ +ailayer_t *ailayer_leaky_relu_f32_default(ailayer_leaky_relu_f32_t *layer, ailayer_t *input_layer); + +#endif // AILAYER_LEAKY_RELU_DEFAULT diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_relu_default.c b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_relu_default.c new file mode 100644 index 0000000..56b2b2d --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_relu_default.c @@ -0,0 +1,39 @@ +/** + * \file basic/default/ailayer/ailayer_relu_default.c + * \version 2.0alpha + * \date 07.12.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/ailayer/ailayer_relu_default.h" + +ailayer_t *ailayer_relu_f32_default(ailayer_relu_f32_t *layer, ailayer_t *input_layer) +{ + layer->dtype = aif32; + + //forward + layer->relu = aimath_f32_default_relu; + + // backward + layer->d_relu = aimath_f32_default_d_relu; + layer->multiply = aimath_f32_default_multiply; + + return ailayer_relu(layer, input_layer); +} diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_relu_default.h b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_relu_default.h new file mode 100644 index 0000000..83e2723 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_relu_default.h @@ -0,0 +1,56 @@ +/** + * \file basic/default/ailayer/ailayer_relu_default.h + * \internal + * \date 07.12.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link ailayer_relu.h ReLU layer \endlink + * + * Hardware independent implementations of the ReLU layer in \link aimath_f32.h F32 \endlink data-type. + * For more information about the ReLU layer refer to ailayer_relu.h. + */ + +#ifndef AILAYER_RELU_DEFAULT +#define AILAYER_RELU_DEFAULT + +#include "basic/base/ailayer/ailayer_relu.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct ailayer_relu ailayer_relu_f32_t; + +/** @brief Initializes and connect a \link ailayer_relu.h ReLU layer \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * Example: Create the layer structure:\n + * \code{.c} + * ailayer_relu_f32_t relu_layer; + * \endcode + * + * Example: Initialize and connect the layer:\n + * \code{.c} + * x = ailayer_relu_f32_default(&relu_layer, x); + * \endcode + * + * @param *layer The layer structure to initialize. + * @param *input_layer The prior layer. + * @return The (successfully) initialized layer structure. + */ +ailayer_t *ailayer_relu_f32_default(ailayer_relu_f32_t *layer, ailayer_t *input_layer); + +#endif // AILAYER_RELU_DEFAULT diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_sigmoid_default.c b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_sigmoid_default.c new file mode 100644 index 0000000..2ed7434 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_sigmoid_default.c @@ -0,0 +1,64 @@ +/** + * \file basic/default/ailayer/ailayer_sigmoid_default.c + * \version 2.0alpha + * \date 12.11.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/ailayer/ailayer_sigmoid_default.h" + +ailayer_t *ailayer_sigmoid_f32_default(ailayer_sigmoid_f32_t *layer, ailayer_t *input_layer) +{ + layer->dtype = aif32; + + //forward + layer->sigmoid = aimath_f32_default_sigmoid; + + // backward + layer->d_sigmoid = aimath_f32_default_d_sigmoid; + layer->multiply = aimath_f32_default_multiply; + + layer->base.get_result_bound = ailayer_sigmoid_get_result_bound_f32_default; + + return ailayer_sigmoid(layer, input_layer); +} + +uint8_t ailayer_sigmoid_get_result_bound_f32_default(const ailayer_t *self, const uint8_t selector, void *result_bound) +{ + float *bound = (float *) result_bound; + + switch(selector){ + case AILAYER_RESULT_LOWER_BOUND: + *bound = 0.0f; + return TRUE; + case AILAYER_RESULT_UPPER_BOUND: + *bound = 1.0f; + return TRUE; + case AILAYER_DELTAS_LOWER_BOUND: + return FALSE; + case AILAYER_DELTAS_UPPER_BOUND: + return FALSE; + default: + #ifdef AIDEBUG_PRINT_ERROR_MESSAGES + printf("\n+++ ERROR: Not defined result bound selector.\n"); + #endif // AIDEBUG_PRINT_ERROR_MESSAGES + return FALSE; + } +} diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_sigmoid_default.h b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_sigmoid_default.h new file mode 100644 index 0000000..1ee255f --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_sigmoid_default.h @@ -0,0 +1,73 @@ +/** + * \file basic/default/ailayer/ailayer_sigmoid_default.h + * \internal + * \date 12.11.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link ailayer_sigmoid.h Sigmoid layer \endlink + * + * Hardware independent implementations of the Sigmoid layer in \link aimath_f32.h F32 \endlink data-type. + * For more information about the Sigmoid layer refer to ailayer_sigmoid.h. + */ + +#ifndef AILAYER_SIGMOID_DEFAULT +#define AILAYER_SIGMOID_DEFAULT + +#include "basic/base/ailayer/ailayer_sigmoid.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct ailayer_sigmoid ailayer_sigmoid_f32_t; + +/** @brief Initializes and connect a \link ailayer_sigmoid.h Sigmoid layer \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * Example: Create the layer structure:\n + * \code{.c} + * ailayer_sigmoid_f32_t sigmoid_layer; + * \endcode + * + * Example: Initialize and connect the layer:\n + * \code{.c} + * x = ailayer_sigmoid_f32_default(&sigmoid_layer, x); + * \endcode + * + * @param *layer The layer structure to initialize. + * @param *input_layer The prior layer. + * @return The (successfully) initialized layer structure. + */ +ailayer_t *ailayer_sigmoid_f32_default(ailayer_sigmoid_f32_t *layer, ailayer_t *input_layer); + +/** @brief Get the value bounds for Sigmoid layers + * + * For the float 32 representation only result bounds are defined. + * + * @param *self Referenced layer structure. + * @param selector Select, which bound value to get. Possible values: + * - AILAYER_RESULT_LOWER_BOUND + * - AILAYER_RESULT_UPPER_BOUND + * - AILAYER_DELTAS_LOWER_BOUND + * - AILAYER_DELTAS_UPPER_BOUND + * @param result_bound Pointer to the memory space, the function writes the requested value to. + * @return + * - TRUE: requested value available and set to result_bound + * - FALSE: requestet value is _not_ available and result_bound is __not__ set + */ +uint8_t ailayer_sigmoid_get_result_bound_f32_default(const ailayer_t *self, const uint8_t selector, void *result_bound); + +#endif // AILAYER_SIGMOID_DEFAULT diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_softmax_default.c b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_softmax_default.c new file mode 100644 index 0000000..d393e3d --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_softmax_default.c @@ -0,0 +1,35 @@ +/** + * \file basic/default/ailayer/ailayer_softmax_default.c + * \version 2.0alpha + * \date 07.12.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/ailayer/ailayer_softmax_default.h" + +ailayer_t *ailayer_softmax_f32_default(ailayer_softmax_f32_t *layer, ailayer_t *input_layer) +{ + layer->dtype = aif32; + + //forward + layer->softmax = aimath_f32_default_softmax; + + return ailayer_softmax(layer, input_layer); +} diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_softmax_default.h b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_softmax_default.h new file mode 100644 index 0000000..cbb95c4 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_softmax_default.h @@ -0,0 +1,56 @@ +/** + * \file basic/default/ailayer/ailayer_softmax_default.h + * \internal + * \date 07.12.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link ailayer_softmax.h Softmax layer \endlink + * + * Hardware independent implementations of the Softmax layer in \link aimath_f32.h F32 \endlink data-type. + * For more information about the Softmax layer refer to ailayer_softmax.h. + */ + +#ifndef AILAYER_SOFTMAX_DEFAULT +#define AILAYER_SOFTMAX_DEFAULT + +#include "basic/base/ailayer/ailayer_softmax.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct ailayer_softmax ailayer_softmax_f32_t; + +/** @brief Initializes and connect an \link ailayer_softmax.h Softmax layer \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * Example: Create the layer structure:\n + * \code{.c} + * ailayer_softmax_f32_t softmax_layer; + * \endcode + * + * Example: Initialize and connect the layer:\n + * \code{.c} + * x = ailayer_softmax_f32_default(&softmax_layer, x); + * \endcode + * + * @param *layer The layer structure to initialize. + * @param *input_layer The prior layer. + * @return The (successfully) initialized layer structure. + */ +ailayer_t *ailayer_softmax_f32_default(ailayer_softmax_f32_t *layer, ailayer_t *input_layer); + +#endif // AILAYER_SOFTMAX_DEFAULT diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_softsign_default.c b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_softsign_default.c new file mode 100644 index 0000000..a766196 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_softsign_default.c @@ -0,0 +1,39 @@ +/** + * \file basic/default/ailayer/ailayer_softsign_default.c + * \version 2.0alpha + * \date 16.03.2021 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/ailayer/ailayer_softsign_default.h" + +ailayer_t *ailayer_softsign_f32_default(ailayer_softsign_f32_t *layer, ailayer_t *input_layer) +{ + layer->dtype = aif32; + + //forward + layer->softsign = aimath_f32_default_softsign; + + // backward + layer->d_softsign = aimath_f32_default_d_softsign; + layer->multiply = aimath_f32_default_multiply; + + return ailayer_softsign(layer, input_layer); +} diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_softsign_default.h b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_softsign_default.h new file mode 100644 index 0000000..06637d8 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_softsign_default.h @@ -0,0 +1,57 @@ +/** + * \file basic/default/ailayer/ailayer_softsign_default.h + * \internal + * \date 16.03.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link ailayer_softsign.h Softsign layer \endlink + * + * Hardware independent implementations of the Softsign layer in \link aimath_f32.h F32 \endlink data-type. + * For more information about the Softsign layer refer to ailayer_softsign.h. + */ + + +#ifndef AILAYER_SOFTSIGN_DEFAULT +#define AILAYER_SOFTSIGN_DEFAULT + +#include "basic/base/ailayer/ailayer_softsign.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct ailayer_softsign ailayer_softsign_f32_t; + +/** @brief Initializes and connect a \link ailayer_softsign.h Softsign layer \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * Example: Create the layer structure:\n + * \code{.c} + * ailayer_softsign_f32_t softsign_layer; + * \endcode + * + * Example: Initialize and connect the layer:\n + * \code{.c} + * x = ailayer_softsign_f32_default(&softsign_layer, x); + * \endcode + * + * @param *layer The layer structure to initialize. + * @param *input_layer The prior layer. + * @return The (successfully) initialized layer structure. + */ +ailayer_t *ailayer_softsign_f32_default(ailayer_softsign_f32_t *layer, ailayer_t *input_layer); + +#endif // AILAYER_SOFTSIGN_DEFAULT diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_tanh_default.c b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_tanh_default.c new file mode 100644 index 0000000..87c3fe6 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_tanh_default.c @@ -0,0 +1,67 @@ + +/** + * \file basic/default/ailayer/ailayer_tanh_default.c + * \version 2.0alpha + * \date 17.03.2021 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/ailayer/ailayer_tanh_default.h" + +ailayer_t *ailayer_tanh_f32_default(ailayer_tanh_f32_t *layer, ailayer_t *input_layer) +{ + layer->dtype = aif32; + + //forward + layer->tanh = aimath_f32_default_tanh; + + // backward + layer->d_tanh = aimath_f32_default_d_tanh; + layer->multiply = aimath_f32_default_multiply; + + layer->base.get_result_bound = ailayer_tanh_get_result_bound_f32_default; + + return ailayer_tanh(layer, input_layer); +} + +uint8_t ailayer_tanh_get_result_bound_f32_default(const ailayer_t *self, const uint8_t selector, void *result_bound) +{ + float *bound = (float *) result_bound; + + switch(selector){ + case AILAYER_RESULT_LOWER_BOUND: + *bound = -1.0f; + return TRUE; + case AILAYER_RESULT_UPPER_BOUND: + *bound = 1.0f; + return TRUE; + case AILAYER_DELTAS_LOWER_BOUND: + return FALSE; + case AILAYER_DELTAS_UPPER_BOUND: + return FALSE; + default: + #ifdef AIDEBUG_PRINT_ERROR_MESSAGES + printf("\n+++ ERROR: Not defined result bound selector.\n"); + #endif // AIDEBUG_PRINT_ERROR_MESSAGES + return FALSE; + } +} + + diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_tanh_default.h b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_tanh_default.h new file mode 100644 index 0000000..9e31ea8 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailayer/ailayer_tanh_default.h @@ -0,0 +1,70 @@ + +/** + * \file basic/default/ailayer/ailayer_tanh_default.h + * \internal + * \date 17.03.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link ailayer_tanh.h Tanh layer \endlink + * + * Hardware independent implementations of the Tanh layer in \link aimath_f32.h F32 \endlink data-type. + * For more information about the Tanh layer refer to ailayer_tanh.h. + */ + +#ifndef AILAYER_TANH_DEFAULT +#define AILAYER_TANH_DEFAULT + +#include "basic/base/ailayer/ailayer_tanh.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct ailayer_tanh ailayer_tanh_f32_t; + +/** @brief Initializes and connect an \link ailayer_tanh.h Tanh layer \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * Example: Create the layer structure:\n + * \code{.c} + * ailayer_tanh_f32_t tanh_layer; + * \endcode + * + * Example: Initialize and connect the layer:\n + * \code{.c} + * x = ailayer_tanh_f32_default(&tanh_layer, x); + * \endcode + * + * Initializes the tanh layer structure with the float 32 default implementation + * + * @param *layer The layer structure to initialize. + * @param *input_layer The prior layer. + * @return The (successfully) initialized layer structure. + */ +ailayer_t *ailayer_tanh_f32_default(ailayer_tanh_f32_t *layer, ailayer_t *input_layer); + +/** @brief Get valid value range of output of tanh layer +* +* Gives the upper and lower bound of the results obtained by interference with the tanh activation function +* +* @param *self the layer to get the upper and lower limit for +* @param *selector the type of the bound (result/deltas, upper/lower) +* @param *result_bound value of the limit, if exists +* @return TRUE or FALSE in case no limit exists +*/ +uint8_t ailayer_tanh_get_result_bound_f32_default(const ailayer_t *self, const uint8_t selector, void *result_bound); + +#endif // AILAYER_TANH_DEFAULT diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailoss/ailoss_crossentropy_default.c b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailoss/ailoss_crossentropy_default.c new file mode 100644 index 0000000..0a0dd99 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailoss/ailoss_crossentropy_default.c @@ -0,0 +1,48 @@ +/** + * \file basic/default/ailoss/ailoss_crossentropy_default.c + * \version 2.0alpha + * \date 14.01.2021 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/ailoss/ailoss_crossentropy_default.h" + + +ailoss_t *ailoss_crossentropy_f32_default(ailoss_crossentropy_f32_t *loss, ailayer_t *input_layer) +{ + loss->dtype = aif32; + + loss->tensor_sub = aimath_f32_default_tensor_sub; + + // Automatically set the right crossentropy function for the output layer. + if(input_layer->layer_type == ailayer_sigmoid_type){ + loss->crossentropy = aimath_f32_default_binary_crossentropy; + } else if (input_layer->layer_type == ailayer_softmax_type){ + loss->crossentropy = aimath_f32_default_categorical_crossentropy; + } else { + // Error: Input layer type not supported + #ifdef AIDEBUG_PRINT_ERROR_MESSAGES + printf("\n!!! Error: Input layer type not supported\n"); + #endif // AIDEBUG_PRINT_ERROR_MESSAGES + return 0; + } + + return ailoss_crossentropy(loss, input_layer); +} diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailoss/ailoss_crossentropy_default.h b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailoss/ailoss_crossentropy_default.h new file mode 100644 index 0000000..67e6421 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailoss/ailoss_crossentropy_default.h @@ -0,0 +1,73 @@ +/** + * \file basic/default/ailoss/ailoss_crossentropy_default.h + * \internal + * \date 14.01.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link ailoss_crossentropy.h Cross-Entropy loss \endlink + * + * Hardware independent implementations of the Cross-Entropy loss in \link aimath_f32.h F32 \endlink data-type. + * For more information about the Cross-Entropy loss refer to ailoss_mse.h. + */ + +#ifndef AILOSS_CROSSENTROPY_DEFAULT +#define AILOSS_CROSSENTROPY_DEFAULT + +#include "basic/base/ailoss/ailoss_crossentropy.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct ailoss_crossentropy ailoss_crossentropy_f32_t; + +/** @brief Initializes and connect a \link ailoss_crossentropy.h Cross-Entropy loss \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * The labels must me either binary (when the output layer is a Sigmoid layer), for example + * @f[ + * \left( \begin{array}{ccc} 1 & 0 & 0 & 1 \\ 1 & 1 & 1 & 0 \\ 0 & 0 & 1 & 0 \end{array}\right) + * @f] + * + * or row wise one-hot encoded (when the output layer is a Softmax layer), for example + * @f[ + * \left( \begin{array}{ccc} 0 & 0 & 0 & 1 \\ 1 & 0 & 0 & 0 \\ 0 & 0 & 1 & 0 \end{array}\right) + * @f] + * + * If you want to provide labels as integers, please use ailoss_crossentropy_sparse8_f32_default() loss. + * + * Example: Create the loss structure:\n + * \code{.c} + * ailoss_crossentropy_f32_t crossentropy_loss; + * \endcode + * + * Example: Initialize and connect the loss to the layer structure:\n + * \code{.c} + * aimodel_t model; + * ... + * model.output_layer = ailayer_sigmoid_f32_default(&sigmoid_layer, x); + * + * model.loss = ailoss_crossentropy_f32_default(&crossentropy_loss, model.output_layer); + * \endcode + * + * @param *loss The loss structure to initialize. + * @param *input_layer The output layer of the model (**Must be either a Sigmoid or a Softmax layer!**). + * @return The (successfully) initialized loss structure. + */ +ailoss_t *ailoss_crossentropy_f32_default(ailoss_crossentropy_f32_t *loss, ailayer_t *input_layer); + +#endif // AILOSS_CROSSENTROPY_DEFAULT + diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailoss/ailoss_mse_default.c b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailoss/ailoss_mse_default.c new file mode 100644 index 0000000..7af3d78 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailoss/ailoss_mse_default.c @@ -0,0 +1,36 @@ +/** + * \file basic/default/ailoss/ailoss_mse_default.c + * \version 2.0alpha + * \date 28.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/ailoss/ailoss_mse_default.h" + + +ailoss_t *ailoss_mse_f32_default(ailoss_mse_f32_t *loss, ailayer_t *input_layer) +{ + loss->dtype = aif32; + + loss->tensor_sub = aimath_f32_default_tensor_sub; + loss->norm_squared = aimath_f32_default_norm_squared; + + return ailoss_mse(loss, input_layer); +} diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailoss/ailoss_mse_default.h b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailoss/ailoss_mse_default.h new file mode 100644 index 0000000..a294287 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/ailoss/ailoss_mse_default.h @@ -0,0 +1,58 @@ +/** + * \file basic/default/ailoss/ailoss_mse_default.h + * \internal + * \date 28.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link ailoss_mse.h Mean Squared Error loss \endlink + * + * Hardware independent implementations of the Mean Squared Error loss in \link aimath_f32.h F32 \endlink data-type. + * For more information about the Mean Squared Error loss refer to ailoss_mse.h. + */ + +#ifndef AILOSS_MSE_DEFAULT +#define AILOSS_MSE_DEFAULT + +#include "basic/base/ailoss/ailoss_mse.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct ailoss_mse ailoss_mse_f32_t; + +/** @brief Initializes and connect a \link ailoss_mse.h Mean Squared Error loss \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * Example: Create the loss structure:\n + * \code{.c} + * ailoss_mse_f32_t mse_loss; + * \endcode + * + * Example: Initialize and connect the loss to the layer structure:\n + * \code{.c} + * aimodel_t model; + * ... + * model.loss = ailoss_mse_f32_default(&mse_loss, model.output_layer); + * \endcode + * + * @param *loss The loss structure to initialize. + * @param *input_layer The output layer of the model. + * @return The (successfully) initialized loss structure. + */ +ailoss_t *ailoss_mse_f32_default(ailoss_mse_f32_t *loss, ailayer_t *input_layer); + +#endif // AILOSS_MSE_DEFAULT diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/aimath/aimath_f32_default.c b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/aimath/aimath_f32_default.c new file mode 100644 index 0000000..04c97d2 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/aimath/aimath_f32_default.c @@ -0,0 +1,551 @@ +/** + * \file basic/default/aimath/aimath_f32_default.c + * \version 2.0alpha + * \date 25.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/aimath/aimath_f32_default.h" +#include + + +void aimath_f32_default_linear(const aitensor_t *a, const aitensor_t *b, const aitensor_t *c, aitensor_t *result) +{ + uint16_t i, j, k; + float sum; + + float *a_data = (float *) a->data; + float *b_data = (float *) b->data; + float *c_data = c != 0 ? (float *) c->data : 0; + float *result_data = (float *) result->data; + +#ifdef SHAPE_CHECK + if(a->shape[1] != b->shape[0]) + { + LOG_E("MatMul input shapes doesn't match.\n"); + return; + } + if(a->shape[0] != result->shape[0] || b->shape[1] != result->shape[1]) + { + LOG_E("MatMul output shape doesn't match.\n"); + return; + } +#endif + + for(i = 0; i < a->shape[0]; i++) + { + for(j = 0; j < b->shape[1]; j++) + { + sum = 0.0f; + for(k = 0; k < a->shape[1]; k++) + { + sum += a_data[i*a->shape[1] + k] * b_data[k*b->shape[1] + j]; + } + if(c != 0){ + // Bias add + sum += c_data[j]; + } + result_data[i*b->shape[1] + j] = sum; + } + } + return; +} + +void aimath_f32_default_mat_mul(const aitensor_t *a, const aitensor_t *b, aitensor_t *result){ + aimath_f32_default_linear(a, b, 0, result); +} + +void aimath_f32_default_multiply(const aitensor_t *a, const aitensor_t *b, aitensor_t *result) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(a); i++) + { + ((float *) result->data)[i] = ((float *) a->data)[i] * ((float *) b->data)[i]; + } + return; +} + +void aimath_f32_default_divide(const aitensor_t *a, const aitensor_t *b, aitensor_t *result) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(a); i++) + { + ((float *) result->data)[i] = ((float *) a->data)[i] / ((float *) b->data)[i]; + } + return; +} + +void aimath_f32_default_scalar_mul(const void *scalar, const aitensor_t *a, aitensor_t *result) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(a); i++) + { + ((float *) result->data)[i] = *((float *) scalar) * ((float *) a->data)[i]; + } + return; +} + +void aimath_f32_default_scalar_add(const void *scalar, const aitensor_t *a, aitensor_t *result) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(a); i++) + { + ((float *) result->data)[i] = *((float *) scalar) + ((float *) a->data)[i]; + } + return; +} + +/** +* Add 2D tensors +*/ +void aimath_f32_default_tensor_add(const aitensor_t *a, const aitensor_t *b, aitensor_t *result) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(a); i++) + { + ((float *) result->data)[i] = ((float *) a->data)[i] + ((float *) b->data)[i]; + } + return; +} + +void aimath_f32_default_tensor_sub(const aitensor_t *a, const aitensor_t *b, aitensor_t *result) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(a); i++) + { + ((float *) result->data)[i] = ((float *) a->data)[i] - ((float *) b->data)[i]; + } + return; +} + +// only for 2D tensors +// a: f32 +// b: u8 +// result: f32 +void aimath_f32_default_tensor_sub_sparse8(const aitensor_t *a, const aitensor_t *b, aitensor_t *result) +{ + uint32_t i, index; + for(i = 0; i < a->shape[0]; i++) + { + index = a->shape[1] * i + ((uint8_t *) b->data)[i]; + ((float *) result->data)[index] = ((float *) a->data)[index] - 1.0f; + } + return; +} + +void aimath_f32_default_copy_tensor(const aitensor_t *from, aitensor_t *to) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(from); i++) + { + ((float *) to->data)[i] = ((float *) from->data)[i]; + } + return; +} + +void aimath_f32_default_transpose_vector(aitensor_t *vector) +{ + uint16_t temp; + temp = vector->shape[0]; + vector->shape[0] = vector->shape[1]; + vector->shape[1] = temp; + return; +} + +void aimath_f32_default_norm_squared(const aitensor_t *x, void *result) +{ + uint32_t i; + + *((float *) result) = 0.0f; + + for(i = 0; i < x->shape[0] * x->shape[1]; i++) + { + *((float *) result) += ((float *) x->data)[i] * ((float *) x->data)[i]; + } + return; +} + +void aimath_f32_default_sum(const aitensor_t *x, void *result) +{ + uint32_t i; + + *((float *) result) = 0.0f; + + for(i = 0; i < x->shape[0] * x->shape[1]; i++) + { + *((float *) result) += ((float *) x->data)[i]; + } + return; +} + +void aimath_f32_default_min(const aitensor_t *x, void *result) +{ + uint32_t i; + float min_value = FLT_MAX; + + for(i = 0; i < aimath_tensor_elements(x); i++) + { + if(((float *) x->data)[i] < min_value){ + min_value = ((float *) x->data)[i]; + } + } + *((float *) result) = min_value; + return; +} + + +void aimath_f32_default_max(const aitensor_t *x, void *result) +{ + uint32_t i; + float max_value = -FLT_MAX; + + for(i = 0; i < aimath_tensor_elements(x); i++) + { + if(((float *) x->data)[i] > max_value){ + max_value = ((float *) x->data)[i]; + } + } + *((float *) result) = max_value; + return; +} + + +void aimath_f32_default_sigmoid(const aitensor_t *x, aitensor_t *result) +{ + uint32_t i; + + for(i = 0; i < aimath_tensor_elements(x); i++) + { + ((float *) result->data)[i] = 1.0f / (1.0f + expf(- ((float *) x->data)[i])); + } + return; +} + +void aimath_f32_default_d_sigmoid(const aitensor_t *sigmoid_x, aitensor_t *result) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(sigmoid_x); i++) + { + // sigmoid'(x) = sigmoid(x) * (1 - sigmoid(x)) + ((float *) result->data)[i] = ((float *) sigmoid_x->data)[i] * (1.0f - ((float *) sigmoid_x->data)[i]); + } + return; +} + +void aimath_f32_default_tanh(const aitensor_t *x, aitensor_t *result) +{ + uint32_t i; + float temp; + for(i = 0; i < aimath_tensor_elements(x); i++) + { + temp = expf(((float *) x->data)[i]); + ((float *) result->data)[i] = (temp - (1.0f/temp)) / (temp + (1.0f/temp)); + } + return; +} + +void aimath_f32_default_d_tanh(const aitensor_t *tanh_x, aitensor_t *result) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(tanh_x); i++) + { + // tanh'(x) = 1 - (tanh(x))^2 + ((float *) result->data)[i] = 1.0f - (((float *) tanh_x->data)[i] * ((float *) tanh_x->data)[i]); + } + return; +} + +void aimath_f32_default_relu(const aitensor_t *x, aitensor_t *result) +{ + uint32_t i; + + for(i = 0; i < aimath_tensor_elements(x); i++) + { + ((float *) result->data)[i] = ((float *) x->data)[i] > 0.0f ? ((float *) x->data)[i] : 0.0f; + } + return; +} + +void aimath_f32_default_d_relu(const aitensor_t *x, aitensor_t *result) +{ + uint32_t i; + + for(i = 0; i < aimath_tensor_elements(x); i++) + { + ((float *) result->data)[i] = ((float *) x->data)[i] >= 0.0f ? 1.0f : 0.0f; + } + return; +} + +void aimath_f32_default_leaky_relu(const aitensor_t *x, const void *alpha, aitensor_t *result) +{ + uint32_t i; + + for(i = 0; i < aimath_tensor_elements(x); i++) + { + ((float *) result->data)[i] = ((float *) x->data)[i] >= 0.0f ? ((float *) x->data)[i] : ((float *) x->data)[i] * *((float *) alpha); + } + return; +} + +void aimath_f32_default_d_leaky_relu(const aitensor_t *x, const void *alpha, aitensor_t *result) +{ + uint32_t i; + + for(i = 0; i < aimath_tensor_elements(x); i++) + { + ((float *) result->data)[i] = ((float *) x->data)[i] >= 0.0f ? 1.0f : *((float *) alpha); + } + return; +} + +void aimath_f32_default_elu(const aitensor_t *x, const void *alpha, aitensor_t *result) +{ + uint32_t i; + + for(i = 0; i < aimath_tensor_elements(x); i++) + { + ((float *) result->data)[i] = ((float *) x->data)[i] > 0.0f ? ((float *) x->data)[i] : (*((float *) alpha) * (exp(((float *) x->data)[i]) - 1.0f)); + } + return; +} + +void aimath_f32_default_d_elu(const aitensor_t *x, const void *alpha, aitensor_t *result) +{ + uint32_t i; + + for(i = 0; i < aimath_tensor_elements(x); i++) + { + ((float *) result->data)[i] = ((float *) x->data)[i] > 0.0f ? 1.0f : (*((float *) alpha) * expf(((float *) x->data)[i])); + } + return; +} + +void aimath_f32_default_softmax(const aitensor_t *x, aitensor_t *result) +{ + uint32_t i, j; + float max; + float exp_sum; + + float *x_data = (float *) x->data; + float *result_data = (float *) result->data; + + // Multiplier for array index calculation + uint16_t multiplier = 1; + for(i = x->dim - 1; i >= 1; i--){ + multiplier *= x->shape[i]; + } + + // Do for every dataset. (0 is batch dimension) + for(i = 0; i < x->shape[0]; i++){ + // calc max value for numeric stability + max = x_data[0]; + for(j = 0; j < multiplier; j++) + { + if(x_data[i * multiplier + j] > max) max = x_data[i * multiplier + j]; + } + // calc exp functions + exp_sum = 0.0f; + for(j = 0; j < multiplier; j++) + { + result_data[i * multiplier + j] = aimath_f32_default_expf_fast(x_data[i * multiplier + j] - max); + exp_sum += result_data[i * multiplier + j]; + } + //calc softmax + for(j = 0; j < multiplier; j++) + { + result_data[i * multiplier + j] = result_data[i * multiplier + j] / exp_sum; + } + } + return; +} + +void aimath_f32_default_softsign(const aitensor_t *x, aitensor_t *result) +{ + uint32_t i; + + for(i = 0; i < aimath_tensor_elements(x); i++) + { + ((float *) result->data)[i] = ((float *) x->data)[i] / (1.0f + fabs(((float *) x->data)[i])); + } + return; +} + +void aimath_f32_default_d_softsign(const aitensor_t *x, aitensor_t *result) +{ + uint32_t i; + + for(i = 0; i < aimath_tensor_elements(x); i++) + { + ((float *) result->data)[i] = 1.0f / pow((1.0f + abs(((float *) x->data)[i])), 2); + } + return; +} + +// predicted data: f32 +// target_data: f32 +void aimath_f32_default_binary_crossentropy(const aitensor_t *predicted_data, const aitensor_t *target_data, void *result) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(predicted_data); i++) + { + if(((float *) target_data->data)[i] != 0){ + *((float *) result) -= ((float *) target_data->data)[i] * log(((float *) predicted_data->data)[i]) + + (1.0f - ((float *) target_data->data)[i]) * log(1.0f - ((float *) predicted_data->data)[i]); + } + } + return; +} + +// predicted data: f32 +// target_data: f32 +void aimath_f32_default_categorical_crossentropy(const aitensor_t *predicted_data, const aitensor_t *target_data, void *result) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(predicted_data); i++) + { + if(((float *) target_data->data)[i] != 0){ + *((float *) result) -= ((float *) target_data->data)[i] * log(((float *) predicted_data->data)[i]); + } + } + return; +} + +// 2D tensors only +// predicted data: f32 +// target_data: u8 +void aimath_f32_default_categorical_crossentropy_sparse8(const aitensor_t *predicted_data, const aitensor_t *target_data, void *result) +{ + uint32_t i, index; + for(i = 0; i < target_data->shape[0]; i++) + { + index = i * predicted_data->shape[1] + ((uint8_t *) target_data->data)[i]; + *((float *) result) -= log(((float *) predicted_data->data)[index]); + } + return; +} + +void aimath_f32_default_sqrt(const aitensor_t *x, aitensor_t *result) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(x); i++) + { + ((float *) result->data)[i] = sqrt(((float *) x->data)[i]); + } + return; +} + +void aimath_f32_default_zero_tensor(aitensor_t *tensor) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(tensor); i++) + { + ((float *) tensor->data)[i] = 0.0f; + } + return; +} + +void aimath_f32_default_init_zeros(aitensor_t *tensor) +{ + aimath_f32_default_zero_tensor(tensor); + return; +} + +void aimath_f32_default_tensor_init_uniform(aitensor_t *tensor, float from, float to) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(tensor); i++) + { + ((float *) tensor->data)[i] = ((float) rand() / (float) RAND_MAX) * (to - from) + from; + } + return; +} + +/* Glorot Uniform weight Initialization + * + * Glorot uniform initializer, also called Xavier uniform initializer. + * + * Glorot et al., 2010 + * + * \ref http://proceedings.mlr.press/v9/glorot10a/glorot10a.pdf + * \ref https://prateekvishnu.medium.com/xavier-and-he-normal-he-et-al-initialization-8e3d7a087528 + */ +void aimath_f32_default_init_glorot_uniform(aitensor_t *tensor) +{ + float fan_in, fan_out, fan_avg; + if(tensor->dim == 2) + { + fan_in = tensor->shape[0]; + fan_out = tensor->shape[1]; + } + else if(tensor->dim == 4) + { + fan_in = tensor->shape[1] * tensor->shape[2] * tensor->shape[3]; // In channel * kernel_elems + fan_out = tensor->shape[0] * tensor->shape[2] * tensor->shape[3]; // Out channel * kernel_elems + } + + fan_avg = (fan_in + fan_out) / 2.0f; + float r = sqrt(3.0f / fan_avg); + aimath_f32_default_tensor_init_uniform(tensor, -r, r); +} + +/* He Uniform weight Initialization + * + * He et al, 2015 + * + * \ref http://arxiv.org/abs/1502.01852 + * \ref https://prateekvishnu.medium.com/xavier-and-he-normal-he-et-al-initialization-8e3d7a087528 + */ +void aimath_f32_default_init_he_uniform(aitensor_t *tensor) +{ + float fan_in, fan_avg; + if(tensor->dim == 2) + { + fan_in = tensor->shape[0]; + } + else if(tensor->dim == 4) + { + fan_in = tensor->shape[1] * tensor->shape[2] * tensor->shape[3]; // In channel * kernel_elems + } + + fan_avg = fan_in / 2.0f; + float r = sqrt(3.0f / fan_avg); + aimath_f32_default_tensor_init_uniform(tensor, -r, r); +} + +//Info(?): http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.9.4508&rep=rep1&type=pdf +//ToDo: Mit dem in der Quelle erwhnten Parameter c rumspielen +float aimath_f32_default_expf_fast(float x) +{ + if(x > 80.0f){ + x = 80.0f; + } + else if(x < -80.0f){ + x = -80.0f; + } + + union { float f; int32_t x; } u; + //max-values for a ca. [-88 , 88] + // a = 2^23 / ln(2) ; b = 127 * 2^23 + // x_int32 = a * x_float + b + u.x = (int32_t) (12102203.0f * x + 1064631197.0f); + return u.f; + +} diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/aimath/aimath_f32_default.h b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/aimath/aimath_f32_default.h new file mode 100644 index 0000000..043209f --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/aimath/aimath_f32_default.h @@ -0,0 +1,1225 @@ +/** + * \file basic/default/aimath/aimath_f32_default.h + * \internal + * \date 25.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Math functions for \link aimath_f32.h F32 \endlink data type, default implementation + * + * These functions can be used when no hardware specific implementation is available. + */ + +#ifndef AIMATH_F32_DEFAULT +#define AIMATH_F32_DEFAULT + +#include +#include +#include +#include + +#include "basic/base/aimath/aimath_f32.h" + +/** @brief Performs a matrix multiplication of \link aimath_f32.h F32 \endlink matrices a and b and adds a vector c to each row + * + * The addition of the horizontal vector c is performed via broadcast, i.e. element wise in each column + * Mathematically this broadcast is equal to multiplying c with an vertical vector (with the same number of elements as c) + * and adding the result to a * b + * + * @f[ + * result = a \cdot b + \left( \begin{array}{c} + 1 \\ + 1 \\ + \vdots \\ + 1 \\ + \end{array}\right) \cdot c + * @f] + * + * Example: + * @f[ + * a = \left( \begin{array}{rrr} + 1 & 2 & 3 \\ + 4 & 5 & 6 \\ + 7 & 8 & 9 + \end{array}\right) + * @f] + * + * @f[ + * b = \left( \begin{array}{rr} + 1 & 0 \\ + 0 & 1 \\ + 0 & 0 + \end{array}\right) + * @f] + * + * @f[ + * c = \left( \begin{array}{rr} + 2 & 5 + \end{array}\right) + * @f] + * + * @f[ + * result = a \cdot b + \left( \begin{array}{r} 1 \\ + 1 \\ + 1 \\ + \end{array}\right) \cdot c + * @f] + * + * @f[ + * = \left( \begin{array}{rr} + 1 & 2 \\ + 4 & 5 \\ + 7 & 8 + \end{array}\right) + + \left( \begin{array}{rr} + 2 & 5 \\ + 2 & 5 \\ + 2 & 5 + \end{array}\right) + * @f] + * + * @details + * + * @f[ + * = \left( \begin{array}{rr} 3 & 7 \\ + 6 & 10 \\ + 9 & 13 + \end{array}\right) + * @f] + * + * Example: + * \code{.c} + * uint16_t a_shape[2] = {3, 3}; + * float a_data[3*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f, + * 7.0f, 8.0f, 9.0f}; + * aitensor_t a = AITENSOR_2D_F32(a_shape, a_data); + * + * uint16_t b_shape[2] = {3, 2}; + * float b_data[3*2] = {1.0f, 0.0f, + * 0.0f, 1.0f, + * 0.0f, 0.0f}; + * aitensor_t b = AITENSOR_2D_F32(b_shape, b_data); + * + * uint16_t c_shape[2] = {1, 2}; + * float c_data[1*2] = {2.0f, 5.0f}; + * aitensor_t c = AITENSOR_2D_F32(c_shape, c_data); + * + * uint16_t result_shape[2] = {3, 2}; + * float result_data[3*2]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_linear(&a, &b, &c, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *a Q31 matrix a (2D tensor of shape [N x K]) + * @param *b Q31 matrix b (2D tensor of shape [K x M]) + * @param *c Q31 vector c (2D tensor of shape [1 x M]) + * @param *result Resulting Q31 matrix (2D tensor of shape [N x M]) + */ +void aimath_f32_default_linear(const aitensor_t *a, const aitensor_t *b, const aitensor_t *c, aitensor_t *result); + +/** @brief Performs a matrix multiplication of \link aimath_f32.h F32 \endlink matrices a and b + * + * @f[ + * result = a \cdot b + * @f] + * + * Example: + * \code{.c} + * uint16_t a_shape[2] = {3, 3}; + * float a_data[3*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f, + * 7.0f, 8.0f, 9.0f}; + * aitensor_t a = AITENSOR_2D_F32(a_shape, a_data); + * + * uint16_t b_shape[2] = {3, 2}; + * float b_data[3*2] = {1.0f, 0.0f, + * 0.0f, 1.0f, + * 0.0f, 0.0f}; + * aitensor_t b = AITENSOR_2D_F32(b_shape, b_data); + * + * uint16_t result_shape[2] = {3, 2}; + * float result_data[3*2]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_mat_mul(&a, &b, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *a F32 matrix a (2D tensor of shape [N x K]) + * @param *b F32 matrix b (2D tensor of shape [K x M]) + * @param *result Resulting F32 matrix of the multiplication (2D tensor of shape [N x M]) + */ +void aimath_f32_default_mat_mul(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + +/** @brief Performs an element wise multiplication of \link aimath_f32.h F32 \endlink tensors a and b (Hadamard product) + * + * @f[ + * result = a \circ b + * @f] + * + * Example: + * \code{.c} + * uint16_t a_shape[2] = {2, 3}; + * float a_data[2*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f}; + * aitensor_t a = AITENSOR_2D_F32(a_shape, a_data); + * + * uint16_t b_shape[2] = {2, 3}; + * float b_data[2*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f}; + * aitensor_t b = AITENSOR_2D_F32(b_shape, b_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_multiply(&a, &b, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *a F32 tensor a (N-D tensor) + * @param *b F32 tensor b (N-D tensor) + * @param *result Resulting F32 tensor of the element wise multiplication (N-D tensor) + */ +void aimath_f32_default_multiply(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + +/** @brief Performs an element wise division of \link aimath_f32.h F32 \endlink tensors a and b (Hadamard division) + * + * @f[ + * result = a \oslash b + * @f] + * + * Example: + * \code{.c} + * uint16_t a_shape[2] = {2, 3}; + * float a_data[2*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f}; + * aitensor_t a = AITENSOR_2D_F32(a_shape, a_data); + * + * uint16_t b_shape[2] = {2, 3}; + * float b_data[2*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f}; + * aitensor_t b = AITENSOR_2D_F32(b_shape, b_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_divide(&a, &b, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *a F32 tensor a (N-D tensor) + * @param *b F32 tensor b (N-D tensor) + * @param *result Resulting F32 tensor of the element wise division (N-D tensor) + */ +void aimath_f32_default_divide(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + +/** @brief Performs a scalar multiplication (scaling) of \link aimath_f32.h F32 \endlink tensor a and a scalar + * + * @f[ + * result = scalar \cdot a + * @f] + * + * Example: + * \code{.c} + * uint16_t a_shape[2] = {2, 3}; + * float a_data[2*3] = {0.0f, 1.0f, 2.0f, + * 3.0f, 4.0f, 5.0f}; + * aitensor_t a = AITENSOR_2D_F32(a_shape, a_data); + * + * float scalar = 0.1f; + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_scalar_mul(&scalar, &a, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *scalar Scalar (type aiscalar_f32_t / float) + * @param *a F32 tensor a (N-D tensor) + * @param *result Resulting F32 tensor of the scalar multiplication (N-D tensor) + */ +void aimath_f32_default_scalar_mul(const void *scalar, const aitensor_t *a, aitensor_t *result); + +/** @brief Performs an element wise addition of a scalar to a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result = a + \left( \begin{array}{ccc} 1 & \ldots & 1 \\ + \vdots & \ddots & \vdots \\ + 1 & \ldots & 1 + \end{array}\right) \cdot scalar + * @f] + * + * Example: + * \code{.c} + * uint16_t a_shape[2] = {2, 3}; + * float a_data[2*3] = {0.0f, 1.0f, 2.0f, + * 3.0f, 4.0f, 5.0f}; + * aitensor_t a = AITENSOR_2D_F32(a_shape, a_data); + * + * float scalar = 0.1f; + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_scalar_add(&scalar, &a, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *scalar Scalar (type aiscalar_f32_t / float) + * @param *a F32 tensor a (N-D tensor) + * @param *result Resulting F32 tensor of the element wise scalar addition (N-D tensor) + */ +void aimath_f32_default_scalar_add(const void *scalar, const aitensor_t *a, aitensor_t *result); + +/** @brief Performs an element wise addition of \link aimath_f32.h F32 \endlink tensors a and b + * + * @f[ + * result = a + b + * @f] + * + * Example: + * \code{.c} + * uint16_t a_shape[2] = {2, 3}; + * float a_data[2*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f}; + * aitensor_t a = AITENSOR_2D_F32(a_shape, a_data); + * + * uint16_t b_shape[2] = {2, 3}; + * float b_data[2*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f}; + * aitensor_t b = AITENSOR_2D_F32(b_shape, b_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_tensor_add(&a, &b, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *a F32 tensor a (N-D tensor) + * @param *b F32 tensor b (N-D tensor) + * @param *result Resulting F32 tensor of the element wise addition (N-D tensor) + */ +void aimath_f32_default_tensor_add(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + +/** @brief Performs a element wise subtraction of \link aimath_f32.h F32 \endlink tensors a and b + * + * @f[ + * result = a - b + * @f] + * + * Example: + * \code{.c} + * uint16_t a_shape[2] = {2, 3}; + * float a_data[2*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f}; + * aitensor_t a = AITENSOR_2D_F32(a_shape, a_data); + * + * uint16_t b_shape[2] = {2, 3}; + * float b_data[2*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f}; + * aitensor_t b = AITENSOR_2D_F32(b_shape, b_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_tensor_sub(&a, &b, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *a F32 tensor a (N-D tensor) + * @param *b F32 tensor b (N-D tensor) + * @param *result Resulting F32 tensor of the element wise subtraction (N-D tensor) + */ +void aimath_f32_default_tensor_sub(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + +/** @brief Performs a subtraction between a \link aimath_f32.h F32 \endlink matrix a and a \link aimath_u8.h U8 \endlink sparse matrix b + * + * This function can subtract a row wise one-hot encoded matrix in sparse representation + * (just the integer index of the 1 is stored) from a normal F32 matrix a. + * + * For example the matrix + * @f[ + * \left( \begin{array}{ccc} 0 & 0 & 0 & 1 \\ 1 & 0 & 0 & 0 \\ 0 & 0 & 1 & 0 \end{array}\right) + * @f] + * in sparse representation is + * @f[ + * \left( \begin{array}{ccc} 3 \\ 0 \\ 2 \end{array}\right) + * @f] + * + * The result is then calculated as + * @f[ + * result_{ij} = \begin{cases} + a_{ij} - 1 & \text{if } j = b_i\\ + a_{ij} & \text{if } j \neq b_i + \end{cases} + * @f] + * + * Example: + * \code{.c} + * uint16_t a_shape[2] = {2, 3}; + * float a_data[2*3] = {0.2f, 0.1f, 0.7f, + * 0.9f, 0.1f, 0.0f}; + * aitensor_t a = AITENSOR_2D_F32(a_shape, a_data); + * + * uint16_t b_shape[2] = {2, 1}; + * uint8_t b_data[2*1] = {2, + * 0}; + * aitensor_t b = AITENSOR_2D_U8(b_shape, b_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_tensor_sub_sparse8(&a, &b, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *a F32 matrix a (2D tensor of shape [N x M]) + * @param *b U8 sparse matrix b (2D tensor of shape [N x 1]) + * @param *result Resulting F32 tensor of the subtraction (2D tensor of shape [N x M]) + */ +void aimath_f32_default_tensor_sub_sparse8(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + +/** @brief Performs an element wise copy of \link aimath_f32.h F32 \endlink tensors + * + * @f[ + * to \leftarrow from + * @f] + * + * Dimension and shape of from and to tensors have to be the same. + * + * Example: + * \code{.c} + * uint16_t from_shape[2] = {2, 3}; + * float from_data[2*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f}; + * aitensor_t from = AITENSOR_2D_F32(from_shape, from_data); + * + * uint16_t to_shape[2] = {2, 3}; + * float to_data[2*3]; + * aitensor_t to = AITENSOR_2D_F32(to_shape, to_data); + * + * aimath_f32_default_copy_tensor(&from, &to); + * + * print_aitensor(&to); + * \endcode + * + * @param *from F32 tensor to copy from (N-D tensor) + * @param *to F32 tensor to copy to (N-D tensor) + */ +void aimath_f32_default_copy_tensor(const aitensor_t *from, aitensor_t *to); + +/** @brief Transposes a \link aimath_f32.h F32 \endlink vector + * + * The given tensor must be a vector (2D tensor of shape [1 x N] or [N x 1]). + * + * @f[ + * vector \leftarrow vector^T + * @f] + * + * Example: + * \code{.c} + * uint16_t vector_shape[2] = {1, 3}; + * float vector_data[1*3] = {1.0f, 2.0f, 3.0f}; + * aitensor_t vector = AITENSOR_2D_F32(vector_shape, vector_data); + * + * aimath_f32_default_transpose_vector(&vector); + * + * print_aitensor(&vector); + * \endcode + * + * @param *vector F32 vector (2D tensor of shape [1 x N] or [N x 1]) + */ +void aimath_f32_default_transpose_vector(aitensor_t *vector); + +/** @brief Calculates the squared sum of all elements in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result = \sum_i x_{i}^2 + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = {0.0f, 1.0f, 2.0f, + * 3.0f, 4.0f, 5.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * float result; + * + * aimath_f32_default_norm_squared(&x, &result); + * + * print_aiscalar(&result, aif32); + * \endcode + * + * @param *x F32 tensor x (N-D tensor) + * @param *result Scalar result (type aiscalar_f32_t / float) + */ +void aimath_f32_default_norm_squared(const aitensor_t *x, void *result); + +/** @brief Calculates the sum of all elements in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result = \sum_i x_{i} + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = {0.0f, 1.0f, 2.0f, + * 3.0f, 4.0f, 5.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * float result; + * + * aimath_f32_default_sum(&x, &result); + * + * print_aiscalar(&result, aif32); + * \endcode + * + * @param *x F32 tensor x (N-D tensor) + * @param *result Scalar result (type aiscalar_f32_t / float) + */ +void aimath_f32_default_sum(const aitensor_t *x, void *result); + +/** @brief Identifies the minimum value in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result = min(x) + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = {0.0f, 1.0f, 2.0f, + * 3.0f, 4.0f, 5.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * float result; + * + * aimath_f32_default_min(&x, &result); + * + * print_aiscalar(&result, aif32); + * \endcode + * + * @param *x F32 tensor x to get the minimum value of (N-D tensor) + * @param *result Scalar result (type aiscalar_f32_t / float) + */ +void aimath_f32_default_min(const aitensor_t *x, void *result); + +/** @brief Identifies the maximum value in a \link aimath_f32.h F32 \endlink tensor + * + * @details + * + * @f[ + * result = max(x) + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = {0.0f, 1.0f, 2.0f, + * 3.0f, 4.0f, 5.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * float result; + * + * aimath_f32_default_max(&x, &result); + * + * print_aiscalar(&result, aif32); + * \endcode + * + * @param *x F32 tensor x to get the maximum value of (N-D tensor) + * @param *result Scalar result (type aiscalar_f32_t / float) + */ +void aimath_f32_default_max(const aitensor_t *x, void *result); + +/** @brief Calculates the sigmoid of each element in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{i} = \sigma(x_{i}) = \frac{1}{1 + e^{-x_{i}}} + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_sigmoid(&x, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *x F32 tensor to calculate the sigmoid from (N-D tensor) + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_sigmoid(const aitensor_t *x, aitensor_t *result); + +/** @brief Calculates the derivative sigmoid of each element in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{i} = \sigma'(x_{i}) = \sigma(x_{i}) \cdot (1 - \sigma(x_{i})) + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_sigmoid(&x, &result); + * aimath_f32_default_d_sigmoid(&result, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *sigmoid_x F32 tensor with the sigmoid values \f$ \sigma(x_{i}) \f$ (N-D tensor) + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_d_sigmoid(const aitensor_t *sigmoid_x, aitensor_t *result); + +/** @brief Calculates the tanh of each element in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{i} = \tanh(x_{i}) = \frac{e^{x_i} - e^{-x_i}}{e^{x_i} + e^{-x_i}} + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_tanh(&x, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *x F32 tensor to calculate the tanh from (N-D tensor) + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_tanh(const aitensor_t *x, aitensor_t *result); + +/** @brief Calculates the tanh derivative of each element in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{i} = tanh'(x_{i}) = 1 - tanh(x_{i})^2 + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_tanh(&x, &result); + * aimath_f32_default_d_tanh(&result, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *tanh_x F32 tensor with the tanh values \f$ \tanh(x_{i}) \f$ (N-D tensor) + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_d_tanh(const aitensor_t *tanh_x, aitensor_t *result); + +/** @brief Calculates the rectifier (ReLU) value of each element in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{i} = max(0, x_{i}) + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_relu(&x, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *x F32 tensor to calculate the ReLU from (N-D tensor) + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_relu(const aitensor_t *x, aitensor_t *result); + + +/** @brief Calculates the rectifier (ReLU) derivative of each element in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{ij} = \begin{cases} + 0 & \text{if } x_i < 0\\ + 1 & \text{if } x_i \geq 0 + \end{cases} + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_d_relu(&x, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *x F32 tensor to calculate the ReLU derivative from (N-D tensor) + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_d_relu(const aitensor_t *x, aitensor_t *result); + +/** @brief Calculates the leaky rectifier (leaky ReLU) value of each element in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{i} = \begin{cases} + \alpha \cdot x_i & \text{if } x_i < 0 \\ + x_i & \text{if } x_i \geq 0 + \end{cases} + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * float alpha = 0.01f; + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_leaky_relu(&x, &alpha, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *x F32 tensor to calculate the leaky ReLU from (N-D tensor) + * @param *alpha Scalar \f$ \alpha \f$ (type aiscalar_f32_t / float) for the leakage + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_leaky_relu(const aitensor_t *x, const void *alpha, aitensor_t *result); + +/** @brief Calculates the leaky rectifier (leaky ReLU) derivative of each element in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{i} = \begin{cases} + \alpha & \text{if } x_i < 0\\ + 1 & \text{if } x_i \geq 0 + \end{cases} + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * float alpha = 0.01f; + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_d_leaky_relu(&x, &alpha, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *x F32 tensor to calculate the leaky ReLU derivative from (N-D tensor) + * @param *alpha Scalar \f$ \alpha \f$ (type aiscalar_f32_t / float) for the leakage + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_d_leaky_relu(const aitensor_t *x, const void *alpha, aitensor_t *result); + +/** @brief Calculates the exponential rectifier (ELU) value of each element in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{i} = \begin{cases} + \alpha \cdot (e^{x_i} - 1) & \text{if } x_i < 0 \\ + x_i & \text{if } x_i \geq 0 + \end{cases} + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * float alpha = 1.0f; + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_elu(&x, &alpha, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *x F32 tensor to calculate the ELU from (N-D tensor) + * @param *alpha Scalar \f$ \alpha \f$ (type aiscalar_f32_t / float) + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_elu(const aitensor_t *x, const void *alpha, aitensor_t *result); + +/** @brief Calculates the exponential rectifier (ELU) derivative of each element in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{i} = \begin{cases} + \alpha \cdot e^{x_i} & \text{if } x_i < 0\\ + 1 & \text{if } x_i \geq 0 + \end{cases} + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * float alpha = 1.0f; + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_d_elu(&x, &alpha, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *x F32 tensor to calculate the ELU derivative from (N-D tensor) + * @param *alpha Scalar \f$ \alpha \f$ (type aiscalar_f32_t / float) + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_d_elu(const aitensor_t *x, const void *alpha, aitensor_t *result); + +/** @brief Calculates the softmax value of each row of a \link aimath_f32.h F32 \endlink matrix + * + * @f[ + * result_{i} = \frac{e^{x_i}}{\sum_{j=1}^{K} e^{x_j}} + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_softmax(&x, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *x F32 matrix to calculate the softmax from (2D tensor) + * @param *result Resulting F32 matrix (2D tensor) + */ +void aimath_f32_default_softmax(const aitensor_t *x, aitensor_t *result); + +/** @brief Calculates the softsign value of each element in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{i} = \frac {x_i} {1 + |x_i|} + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_softsign(&x, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *x F32 tensor to calculate the softsign from (N-D tensor) + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_softsign(const aitensor_t *x, aitensor_t *result); + +/** @brief Calculates the softsign derivative of each element in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{i} = \frac {x_i} {(1 + |x_i|)^2} + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_d_softsign(&x, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *x F32 tensor to calculate the softsign derivative from (N-D tensor) + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_d_softsign(const aitensor_t *x, aitensor_t *result); + +/** @brief Calculates the binary cross entropy between the \link aimath_f32.h F32 \endlink predicted and the target data + * + * @f[ + * result = - \sum_i (target_i \cdot \log(predicted_i) + (1 - target_i) \cdot \log(1 - predicted_i)) + * @f] + * + * Example: + * \code{.c} + * uint16_t p_shape[2] = {2, 3}; + * float p_data[2*3] = {0.8f, 0.1f, 0.7f, + * 0.2f, 0.3f, 0.0f}; + * aitensor_t p = AITENSOR_2D_F32(p_shape, p_data); + * + * uint16_t t_shape[2] = {2, 3}; + * float t_data[2*3] = {1.0f, 0.0f, 1.0f, + * 0.0f, 0.0f, 0.0f}; + * aitensor_t t = AITENSOR_2D_F32(t_shape, t_data); + * + * float result; + * + * aimath_f32_default_binary_crossentropy(&p, &t, &result); + * + * print_aiscalar(&result, aif32); + * \endcode + * + * @param *predicted_data F32 matrix with the predicted or calculated values (2D tensor) + * @param *target_data F32 matrix with the target data / true values / labels (2D tensor with binary values 0 or 1) + * @param *result Resulting F32 matrix (2D tensor) + */ +void aimath_f32_default_binary_crossentropy(const aitensor_t *predicted_data, const aitensor_t *target_data, void *result); + +/** @brief Calculates the categorical cross entropy between the \link aimath_f32.h F32 \endlink predicted and the target data + * + * @f[ + * result = - \sum_i target_i \cdot \log(predicted_i) + * @f] + * + * Example: + * \code{.c} + * uint16_t p_shape[2] = {2, 3}; + * float p_data[2*3] = {0.2f, 0.1f, 0.7f, + * 0.9f, 0.1f, 0.0f}; + * aitensor_t p = AITENSOR_2D_F32(p_shape, p_data); + * + * uint16_t t_shape[2] = {2, 3}; + * float t_data[2*3] = {0.0f, 0.0f, 1.0f, + * 1.0f, 0.0f, 0.0f}; + * aitensor_t t = AITENSOR_2D_F32(t_shape, t_data); + * + * float result; + * + * aimath_f32_default_categorical_crossentropy(&p, &t, &result); + * + * print_aiscalar(&result, aif32); + * \endcode + * + * @param *predicted_data F32 matrix with the predicted or calculated values (2D tensor) + * @param *target_data F32 matrix with the target data / true values / labels (2D tensor, rows one-hot encoded) + * @param *result Resulting F32 matrix (2D tensor) + */ +void aimath_f32_default_categorical_crossentropy(const aitensor_t *predicted_data, const aitensor_t *target_data, void *result); + +/** @brief Calculates the categorical Cross-Entropy between the \link aimath_f32.h F32 \endlink predicted data and the \link aimath_u8.h U8 \endlink target data in sparse representation + * + * This function can calculate the crossentropy between a row wise one-hot encoded matrix in sparse representation + * (just the integer index of the 1 is stored) and a normal F32 matrix a. + * + * For example the matrix + * @f[ + * \left( \begin{array}{ccc} 0 & 0 & 0 & 1 \\ 1 & 0 & 0 & 0 \\ 0 & 0 & 1 & 0 \end{array}\right) + * @f] + * in sparse representation is + * @f[ + * \left( \begin{array}{ccc} 3 \\ 0 \\ 2 \end{array}\right) + * @f] + * + * The result is then calculated from the one-hot encoded target matrix: + * @f[ + * result = - \sum_i target_{one-hot,i} \cdot \log(predicted_i) + * @f] + * + * Example: + * \code{.c} + * uint16_t p_shape[2] = {2, 3}; + * float p_data[2*3] = {0.2f, 0.1f, 0.7f, + * 0.9f, 0.1f, 0.0f}; + * aitensor_t p = AITENSOR_2D_F32(p_shape, p_data); + * + * uint16_t t_shape[2] = {2, 1}; + * uint8_t t_data[2*1] = {2, + * 0}; + * aitensor_t t = AITENSOR_2D_U8(t_shape, t_data); + * + * float result; + * + * aimath_f32_default_categorical_crossentropy_sparse8(&p, &t, &result); + * + * print_aiscalar(&result, aif32); + * \endcode + * + * @param *predicted_data F32 matrix with the predicted or calculated values (2D tensor of shape [N x M]) + * @param *target_data U8 matrix with the target data / true values / labels (2D tensor of shape [N x 1] with true class indices) + * @param *result Resulting F32 matrix (2D tensor of shape [N x M]) + */ +void aimath_f32_default_categorical_crossentropy_sparse8(const aitensor_t *predicted_data, const aitensor_t *target_data, void *result); + +/** @brief Calculates the element wise square root of a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{i} = \sqrt{x_{i}} + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_sqrt(&x, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *x F32 tensor to calculate the square root from (N-D tensor) + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_sqrt(const aitensor_t *x, aitensor_t *result); + +/** @brief Fills a \link aimath_f32.h F32 \endlink tensor with zeros + * + * @f[ + * tensor_{i} = 0 + * @f] + * + * Example: + * \code{.c} + * uint16_t tensor_shape[2] = {2, 3}; + * float tensor_data[2*3]; + * aitensor_t tensor = AITENSOR_2D_F32(tensor_shape, tensor_data); + * + * aimath_f32_default_zero_tensor(&tensor); + * + * print_aitensor(&tensor); + * \endcode + * + * In the F32 implementation of this function, there is no difference between aimath_f32_default_zero_tensor() + * and aimath_f32_default_init_zeros(). + * + * @param *tensor F32 tensor to set to zero (N-D tensor) + */ +void aimath_f32_default_zero_tensor(aitensor_t *tensor); + +/** @brief Fills a \link aimath_f32.h F32 \endlink tensor with zeros + * + * @f[ + * tensor_{i} = 0 + * @f] + * + * Example: + * \code{.c} + * uint16_t tensor_shape[2] = {2, 3}; + * float tensor_data[2*3]; + * aitensor_t tensor = AITENSOR_2D_F32(tensor_shape, tensor_data); + * + * aimath_f32_default_init_zeros(&tensor); + * + * print_aitensor(&tensor); + * \endcode + * + * In the F32 implementation of this function, there is no difference between aimath_f32_default_zero_tensor() + * and aimath_f32_default_init_zeros(). + * + * @param *tensor F32 tensor to set to zero (N-D tensor) + */ +void aimath_f32_default_init_zeros(aitensor_t *tensor); + +/** @brief Fills a \link aimath_f32.h F32 \endlink tensor with random numbers created from a uniform distribution within given range + * + * @f[ + * tensor_i \in \mathcal{U(from, to)} + * @f] + * + * Example: + * \code{.c} + * uint16_t tensor_shape[2] = {2, 3}; + * float tensor_data[2*3]; + * aitensor_t tensor = AITENSOR_2D_F32(tensor_shape, tensor_data); + * + * aimath_f32_default_tensor_init_uniform(&tensor, -1.5f, 1.5f); + * + * print_aitensor(&tensor); + * \endcode + * + * @param *tensor F32 tensor to initialize with random numbers (N-D tensor) + * @param from Minimum value of the uniform distribution + * @param to Maximum value of the uniform distribution + */ +void aimath_f32_default_tensor_init_uniform(aitensor_t *tensor, float from, float to); + +/** @brief Fills a \link aimath_f32.h F32 \endlink tensor with random numbers uniformly within given range, according to Glorot et al. + * + * @f[ + * fan_{avg} = \frac{fan_{in} + fan_{out}}{2} + * @f] + * @f[ + * r = \sqrt{\frac{3}{fan_{avg}}} + * @f] + * @f[ + * tensor_i \in \mathcal{U(-r, r)} + * @f] + * + * Example: + * \code{.c} + * uint16_t tensor_shape[2] = {2, 3}; + * float tensor_data[2*3]; + * aitensor_t tensor = AITENSOR_2D_F32(tensor_shape, tensor_data); + * + * aimath_f32_default_init_glorot_uniform(&tensor); + * + * print_aitensor(&tensor); + * \endcode + * + * @see Glorot et al., 2010 ( http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf ) + * + * @param *tensor F32 tensor to initialize with random numbers (N-D tensor) + */ +void aimath_f32_default_init_glorot_uniform(aitensor_t *tensor); + +/** @brief Fills a \link aimath_f32.h F32 \endlink tensor with uniformly drawn random numbers within given range, according to He et al. + * + * @f[ + * fan_{avg} = \frac{fan_{in}}{2} + * @f] + * @f[ + * r = \sqrt{\frac{3}{fan_{avg}}} + * @f] + * @f[ + * tensor_i \in \mathcal{U(-r, r)} + * @f] + * + * Example: + * \code{.c} + * uint16_t tensor_shape[2] = {2, 3}; + * float tensor_data[2*3]; + * aitensor_t tensor = AITENSOR_2D_F32(tensor_shape, tensor_data); + * + * aimath_f32_default_init_he_uniform(&tensor); + * + * print_aitensor(&tensor); + * \endcode + * + * @see He et al., 2015 ( https://www.cv-foundation.org/openaccess/content_iccv_2015/html/He_Delving_Deep_into_ICCV_2015_paper.html ) + * + * @param *tensor F32 tensor to initialize with random numbers (N-D tensor) + */ +void aimath_f32_default_init_he_uniform(aitensor_t *tensor); + +/** @brief Fast approximation of the exponential function + * + * @see http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.9.4508&rep=rep1&type=pdf + * + * @param x Input of the exponential function + */ +float aimath_f32_default_expf_fast(const float x); + +#endif // AIMATH_F32_DEFAULT + diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/aiopti/aiopti_adam_default.c b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/aiopti/aiopti_adam_default.c new file mode 100644 index 0000000..0f42a68 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/aiopti/aiopti_adam_default.c @@ -0,0 +1,88 @@ +/** + * \file basic/default/aiopti/aiopti_adam_default.c + * \version 2.0alpha + * \date 12.11.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/aiopti/aiopti_adam_default.h" + +aiopti_t *aiopti_adam_f32_default(aiopti_adam_f32_t *opti) +{ + opti->base.base.dtype = aif32; + + opti->base.base.learning_rate = &opti->learning_rate; + + opti->base.beta1 = &opti->beta1; + opti->base.beta2 = &opti->beta2; + opti->base.eps = &opti->eps; + + opti->base.beta1t = &opti->beta1t; + opti->base.beta2t = &opti->beta2t; + opti->base.one_minus_beta1 = &opti->one_minus_beta1; + opti->base.one_minus_beta2 = &opti->one_minus_beta2; + opti->base.lrt = &opti->lrt; + + // ToDo: Check Initialization + opti->beta1t = opti->beta1; + opti->beta2t = opti->beta2; + opti->one_minus_beta1 = 1.0f - opti->beta1; + opti->one_minus_beta2 = 1.0f - opti->beta2; + //opti->lrt = *((float *) opti->base.base.learning_rate) * sqrtf(1.0f - opti->beta2) / (1.0f - opti->beta1); + + // Set f32 function of optimizer base function begin_step() and end_step() + opti->base.base.begin_step = aiopti_adam_f32_default_begin_step; + opti->base.base.end_step = aiopti_adam_f32_default_end_step; + + // Set f32 math functions of adam optimizer + opti->base.multiply = aimath_f32_default_multiply; + opti->base.divide = aimath_f32_default_divide; + opti->base.tensor_add = aimath_f32_default_tensor_add; + opti->base.tensor_sub = aimath_f32_default_tensor_sub; + opti->base.scalar_mul = aimath_f32_default_scalar_mul; + opti->base.scalar_add = aimath_f32_default_scalar_add; + opti->base.sqrt = aimath_f32_default_sqrt; + + opti->base.zero_tensor = aimath_f32_default_zero_tensor; + + return aiopti_adam(&opti->base); +} + +void aiopti_adam_f32_default_begin_step(aiopti_t *self) +{ + aiopti_adam_t *opti = (aiopti_adam_t *)(self->optimizer_configuration); + + // lr_t = lr * sqrt(1-beta2^t) / (1-beta1^t) + *((float *) opti->lrt) = *((float *) self->learning_rate) * sqrtf(1.0f - *((float *) opti->beta2t)) / (1.0f - *((float *) opti->beta1t)); + return; +} + +void aiopti_adam_f32_default_end_step(aiopti_t *self) +{ + aiopti_adam_t *opti = (aiopti_adam_t *)(self->optimizer_configuration); + + // beta^t <= beta^t * beta; + *((float *) opti->beta1t) = *((float *) opti->beta1t) * *((float *) opti->beta1); + *((float *) opti->beta2t) = *((float *) opti->beta2t) * *((float *) opti->beta2); + *((float *) opti->one_minus_beta1) = 1.0f - *((float *) opti->beta1); + *((float *) opti->one_minus_beta2) = 1.0f - *((float *) opti->beta2); + + return; +} diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/aiopti/aiopti_adam_default.h b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/aiopti/aiopti_adam_default.h new file mode 100644 index 0000000..8103a3a --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/aiopti/aiopti_adam_default.h @@ -0,0 +1,137 @@ +/** + * \file basic/default/aiopti/aiopti_adam_default.h + * \internal + * \date 28.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link aiopti_adam.h Adam optimizer \endlink + * + * Hardware independent implementations of the Adam optimizer in \link aimath_f32.h F32 \endlink data-type. + * For more information about the Adam optimizer refer to aiopti_adam.h. + */ + +#ifndef AIOPTI_ADAM_DEFAULT +#define AIOPTI_ADAM_DEFAULT + +#include "basic/base/aiopti/aiopti_adam.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct aiopti_adam_f32 aiopti_adam_f32_t; + +/** @brief Data-type specific \link aiopti_adam.h Adam optimizer \endlink struct for \link aimath_f32.h F32 \endlink + * + * Adds data fields for the learning rate and the configuration values in \link aimath_f32.h F32 \endlink to the base implementation. + */ +struct aiopti_adam_f32 { + aiopti_adam_t base; /**< Inherited field members from general optimizer struct. */ + + /** @name Optimizer configuration + * @brief Required configuration parameters for the optimizer + * + * These fields have to be configured by the user before calling the initializer function. + */ + ///@{ + aiscalar_f32_t learning_rate; /**< Storage for aiopti.learning_rate scalar in F32 */ + + aiscalar_f32_t beta1; /**< Storage for aiopti_adam.beta1 scalar in F32 */ + aiscalar_f32_t beta2; /**< Storage for aiopti_adam.beta2 scalar in F32 */ + aiscalar_f32_t eps; /**< Storage for aiopti_adam.eps scalar in F32 */ + ///@} + + /** @name Variables for internal computation + * + * These fields are automatically configured in the initializer function. + */ + ///@{ + aiscalar_f32_t beta1t; /**< Storage for aiopti_adam.beta1t scalar in F32 */ + aiscalar_f32_t beta2t; /**< Storage for aiopti_adam.beta2t scalar in F32 */ + aiscalar_f32_t one_minus_beta1; /**< Storage for aiopti_adam.one_minus_beta1 scalar in F32 */ + aiscalar_f32_t one_minus_beta2; /**< Storage for aiopti_adam.one_minus_beta2 scalar in F32 */ + aiscalar_f32_t lrt; /**< Storage for aiopti_adam.lrt scalar in F32 */ + ///@} +}; + + +/** @brief Initializes an \link aiopti_adam.h Adam optimizer \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * Example: Create the optimizer structure:\n + * \code{.c} + * aiopti_adam_f32_t adam_optimizer = { + * .learning_rate = 0.01f, + * + * .beta1 = 0.9f, + * .beta2 = 0.999f, + * .eps = 1e-7f + * }; + * \endcode + * + * Example: Initialize the optimizer:\n + * \code{.c} + * aiopti_t *optimizer; + * + * optimizer = aiopti_adam_f32_default(&adam_optimizer); + * \endcode + * + * @param *opti The optimizer structure to initialize. + * @return The (successfully) initialized optimizer structure. + */ +aiopti_t *aiopti_adam_f32_default(aiopti_adam_f32_t *opti); + +/** @brief \link aimath_f32.h F32 \endlink default implementation of the aiopti.begin_step function for ADAM + * + * *Implementation of aiopti.begin_step.* + * + * The ADAM optimizer needs some modification of the learning rate in every optimization step. + * This function deals with aiscalars and has to be implemented for every data-type individually. + * + * The calculations are:\n + * @f[ + * lr_t \leftarrow lr \cdot \frac{\sqrt{1 - \beta^t_2}} {(1 - \beta_1)^t} + * @f] + * + * This is not primary time critical function, because it only deals with scalars -> No special hardware implementation necessary (but possible). + * + * @param *self The optimizer structure + */ +void aiopti_adam_f32_default_begin_step(aiopti_t *self); + +/** @brief \link aimath_f32.h F32 \endlink default implementation of the aiopti.end_step function for ADAM + * + * *Implementation of aiopti.end_step.* + * + * The ADAM optimizer needs some modification of the learning rate in every optimization step. + * This function deals with aiscalars and has to be implemented for every data-type individually. + * + * The calculations are:\n + * @f[ + * \beta^t_1 \leftarrow \beta^t_1 \cdot \beta_1 + * @f] + * + * @f[ + * \beta^t_2 \leftarrow \beta^t_2 \cdot \beta_2 + * @f] + * + * This is not primary time critical function, because it only deals with scalars -> No special hardware implementation necessary (but possible). + * + * @param *self The optimizer structure + */ +void aiopti_adam_f32_default_end_step(aiopti_t *self); + +#endif // AIOPTI_ADAM_DEFAULT diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/aiopti/aiopti_sgd_default.c b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/aiopti/aiopti_sgd_default.c new file mode 100644 index 0000000..03ee4e3 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/aiopti/aiopti_sgd_default.c @@ -0,0 +1,58 @@ +/** + * \file basic/default/aiopti/aiopti_sgd_default.c + * \version 2.0alpha + * \date 28.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/aiopti/aiopti_sgd_default.h" + +aiopti_t *aiopti_sgd_f32_default(aiopti_sgd_f32_t *opti) +{ + aiopti_t* return_opti; + + opti->base.base.dtype = aif32; + + // Call "constructor" of base "class" + return_opti = aiopti_sgd(&opti->base); + + return_opti->learning_rate = &(opti->learning_rate); + opti->base.momentum = &opti->momentum; + + // Check if a momentum is set and configure the right functions + if (opti->momentum != 0.0f) { + return_opti->update_params = aiopti_sgd_update_params_with_momentum; + return_opti->sizeof_optimem = aiopti_sgd_sizeof_optimem_with_momentum; + return_opti->init_optimem = aiopti_sgd_init_optimem_with_momentum; + } + else { + return_opti->update_params = aiopti_sgd_update_params_without_momentum; + return_opti->sizeof_optimem = aiopti_sgd_sizeof_optimem_without_momentum; + return_opti->init_optimem = aiopti_sgd_init_optimem_without_momentum; + } + + // Set f32 math functions of sgd optimizer + opti->base.zero_tensor = aimath_f32_default_zero_tensor; + opti->base.tensor_add = aimath_f32_default_tensor_add; + opti->base.tensor_sub = aimath_f32_default_tensor_sub; + opti->base.scalar_mul = aimath_f32_default_scalar_mul; + + return return_opti; +} diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/aiopti/aiopti_sgd_default.h b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/aiopti/aiopti_sgd_default.h new file mode 100644 index 0000000..94793dd --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/basic/default/aiopti/aiopti_sgd_default.h @@ -0,0 +1,75 @@ +/** + * \file basic/default/aiopti/aiopti_sgd_default.h + * \internal + * \date 28.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link aiopti_sgd.h Stochastic Gradient Descend optimizer \endlink + * + * Hardware independent implementations of the Stochastic Gradient Descend optimizer in \link aimath_f32.h F32 \endlink data-type. + * For more information about the Stochastic Gradient Descend optimizer refer to aiopti_sgd.h. + */ + +#ifndef AIOPTI_SGD_DEFAULT +#define AIOPTI_SGD_DEFAULT + +#include "basic/base/aiopti/aiopti_sgd.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct aiopti_sgd_f32 aiopti_sgd_f32_t; /**< New data type name for code reduction. */ + +/** @brief Data-type specific \link aiopti_sgd.h SGD optimizer \endlink struct for \link aimath_f32.h F32 \endlink + * + * Adds data fields for the learning rate and the momentum in \link aimath_f32.h F32 \endlink to the base implementation. + */ +struct aiopti_sgd_f32 { + aiopti_sgd_t base; /**< Inherited field members from general aiopti_sgd struct. */ + + aiscalar_f32_t learning_rate; /**< Storage for aiopti.learning_rate scalar in F32 */ + + aiscalar_f32_t momentum; /**< Storage for aiopti_sgd.momentum scalar in F32 */ +}; + +/** @brief Initializes a \link aiopti_sgd.h SGD optimizer \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * You can set the momentum to zero to save memory space. + * + * Example: Create the optimizer structure:\n + * \code{.c} + * aiopti_sgd_f32_t sgd_optimizer = { + * .learning_rate = 0.01f, + * + * .momentum = 0.9f + * }; + * \endcode + * + * Example: Initialize the optimizer:\n + * \code{.c} + * aiopti_t *optimizer; + * + * optimizer = aiopti_sgd_f32_default(&sgd_optimizer); + * \endcode + * + * @param *opti The optimizer structure to initialize. + * @return The (successfully) initialized optimizer structure. + */ +aiopti_t *aiopti_sgd_f32_default(aiopti_sgd_f32_t *opti); + +#endif // AIOPTI_SGD_DEFAULT diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/core/aifes_core.h b/examples/2_MNIST_on_PC/1_MNIST_train/src/core/aifes_core.h new file mode 100644 index 0000000..202005f --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/core/aifes_core.h @@ -0,0 +1,466 @@ +/** + * \file core/aifes_core.h + * \internal + * \date 20.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief AIfES 2 core interface + * + * This interface defines the functional modules of AIfES 2 like model, layer, loss and optimizer. + * + * The structure of an aimodel and its components can look like in the following example for a simple multilayer perceptron - MLP: + * \image html functional_concept.jpg width=800px + */ + +#ifndef AIFES_CORE +#define AIFES_CORE + +#include "aifes_math.h" + +#define TRUE 1 +#define FALSE 0 + +// ToDo: Make enum from the values +#define AILAYER_RESULT_LOWER_BOUND 0 +#define AILAYER_RESULT_UPPER_BOUND 1 +#define AILAYER_DELTAS_LOWER_BOUND 2 +#define AILAYER_DELTAS_UPPER_BOUND 3 + +typedef struct ailayer ailayer_t; +typedef struct ailoss ailoss_t; +typedef struct aimodel aimodel_t; +typedef struct aiopti aiopti_t; + +typedef struct aicore_layertype aicore_layertype_t; +typedef struct aicore_losstype aicore_losstype_t; +typedef struct aicore_optitype aicore_optitype_t; + + +/** @brief Type indicator of the layer + * + * Every layer should have a constant global variable of this type which indicates the layer type. + * + * Example for the dense layer:\n + * \code{.c} + * const aicore_layertype_t ailayer_dense_type_s = { + * #ifdef AIDEBUG_PRINT_MODULE_SPECS + * .name = "Dense", + * .print_specs = ailayer_dense_print_specs + * #else + * .name = 0, + * .print_specs = 0 + * #endif + * }; + * const aicore_layertype_t *ailayer_dense_type = &ailayer_dense_type_s; + * \endcode + * + * Checks for the layer type can look like this:\n + * \code{.c} + * ailayer_t *example_layer; + * if(example_layer->layer_type == ailayer_dense_type){ + * ... + * } + * \endcode + */ +struct aicore_layertype { + const char *name; /**< Name of the layer type (for example "Dense") */ + + /** @brief Set a function to print specs of the layer (for example size, constants) + * + * This function should only be set in the debug mode when prints are required (to save memory otherwise) + * + * @param self The layer + * @param *print A function for printing (for example printf) + */ + void (*print_specs)(const ailayer_t *self, int (*print)(const char *format, ...)); +}; + +/** @brief Type indicator of the loss to check for the loss type + * + * Every loss should have a constant global variable of this type which indicates the loss type.\n + * Example for the mse loss:\n + * + * \code{.c} + * const aicore_losstype_t ailoss_mse_type_s = { + * #ifdef AIDEBUG_PRINT_MODULE_SPECS + * .name = "Mean Squared Error", + * .print_specs = ailoss_mse_print_spec + * #else + * .name = 0, + * .print_specs = 0 + * #endif + * }; + * const aicore_losstype_t *ailoss_mse_type = &ailoss_mse_type_s; + * \endcode + * + * Checks for the loss type can look like this:\n + * \code{.c} + * ailoss_t *example_loss; + * if(example_loss->loss_type == ailoss_mse_type){ + * ... + * } + * \endcode + */ +struct aicore_losstype { + const char *name; /**< Name of the loss type (for example "Mean Squared Error") */ + + /** @brief Set a function to print specs of the loss + * + * This function should only be set in the debug mode when prints are required (to save memory otherwise) + * + * @param self The loss + * @param *print A function for printing (for example printf) + */ + void (*print_specs)(const ailoss_t *self, int (*print)(const char *format, ...)); +}; + +/** @brief Type indicator of the optimizer to check for the optimizer type + * + * Every optimizer should have a constant global variable of this type which indicates the optimizer type.\n + * Example for the adam optimizer:\n + * + * \code{.c} + * const aicore_optitype_t aiopti_adam_type_s = { + * #ifdef AIDEBUG_PRINT_MODULE_SPECS + * .name = "ADAM", + * .print_specs = aiopti_adam_print_specs + * #else + * .name = 0, + * .print_specs = 0 + * #endif + * }; + * const aicore_optitype_t *aiopti_adam_type = &aiopti_adam_type_s; + * \endcode + * + * Checks for the optimizer type can look like this:\n + * \code{.c} + * aiopti_t *example_optimizer; + * if(example_optimizer->optimizer_type == aiopti_adam_type){ + * ... + * } + * \endcode + */ +struct aicore_optitype { + const char *name; /**< Name of the optimizer type (for example "ADAM") */ + + /** @brief Set a function to print specs of the optimizer + * + * This function should only be set in the debug mode when prints are required (to save memory otherwise) + * + * @param self The optimizer + * @param *print A function for printing (for example printf) + */ + void (*print_specs)(const aiopti_t *self, int (*print)(const char *format, ...)); +}; + +/** @brief AIfES artificial neural network model +* +* \image html aimodel.png width=500px +* +* This structure holds the model structure and compute graph of the ANN. (Refer to aifes_core.h for a structural overview)\n +* A aimodel contains several layers that are connected in a list or graph structure.\n +* When the model should be trained, a loss function is required. Otherwise it can be set to NULL. +*/ +struct aimodel { + ailayer_t *input_layer; /**< Input layer of the model that gets the input data. */ + ailayer_t *output_layer; /**< Output layer of the model. The output of a forwad pass is available in the result tensor of this layer. */ + + uint16_t layer_count; /**< Total number of layers of the model (usually autogenerated). */ + uint16_t trainable_params_count; /**< Total number of trainable parameter tensors */ + + ailoss_t *loss; /**< The loss or cost function of the model (only for training). */ +}; + + +/** @brief AIfES layer interface +* +* \image html ailayer.png width=200px +* +* The interface contains the necessary functions and parameters for inference and training on the model. +* (Refer to aifes_core.h for a structural overview) +* +* The call order of the functions for inference:\n +* \code{.c} +* +* for each layer in the model +* calc_result_shape() +* endfor +* +* for each layer in the model +* forward() +* endfor +* // The result of the inference is now in output_layer.result tensor +* \endcode +* +* The call order of the functions for training:\n +* \code{.c} +* +* for each layer in the model +* calc_result_shape() +* endfor +* +* // If the parameters are not already pretrained, a new parameter memory block can be created +* for each layer in the model +* allocate memory of size sizeof_paramem() +* set_paramem() +* endfor +* init_trainable_parameters_model() // Do some weight initialization +* +* for each layer in the model +* allocate memory of size sizeof_trainmem() +* set_trainmem() +* endfor +* +* init_trainmem_model() // For example set the optimem to zero +* for iterations +* for each batch in dataset +* zero_gradients_model() +* +* for each sample in the batch +* for each layer in the model +* forward() +* endfor +* +* calc_delta() +* for each layer in the model (reverse) +* backward() +* endfor +* endfor +* +* update_params_model() +* endfor +* endfor +* \endcode +*/ +struct ailayer { + const aicore_layertype_t *layer_type; /**< Type of the layer (for example ailayer_dense_type) */ + void *layer_configuration; /**< Layer specific configurations (back-link from abstract layer class to implementation) */ + + /** @name Layer connections + * Defines the model graph. + */ + ///@{ + ailayer_t *input_layer; + //ailayer_t *brother_input_layer; /**< (NOT_IN_USE) Chained list if multiple input layer are present else NULL. */ + + ailayer_t *output_layer; + //ailayer_t *brother_output_layer; /**< (NOT_IN_USE) Chained list if multiple output layer are present else NULL. */ + ///@} + + /** @name Inference and training scheduling order (Not in use yet) + * @brief The scheduler can execute the layers along this path. + */ + ///@{ + ailayer_t *next_scheduled; + ailayer_t *prev_scheduled; + ///@} + + aitensor_t result; /**< The result of the forward function is stored here */ + + /** @brief Result and delta min and max values. + * + * Returns the min/max values if the result can only be in certain value ranges (like sigmoid). + * This may be used e.g. for quantization algorithms. Set to NULL if not in use. + * + * @param self The layer + * @param selector Select the requested bound (AILAYER_RESULT_LOWER_BOUND, AILAYER_RESULT_UPPER_BOUND, AILAYER_DELTA_LOWER_BOUND, AILAYER_DELTA_UPPER_BOUND) + * @param result_bound A scalar of aiscalar type in which the bound value can be written + * @return TRUE if bound available else FALSE + */ + uint8_t (*get_result_bound)(const ailayer_t *self, const uint8_t selector, void *result_bound); + + /** @brief Calculate and write the shape to the result tensor. + * + * Made for easy creation of the model (layers can be connected to each other without worrying about the shapes). + * + * @param self The layer + */ + void (*calc_result_shape)(ailayer_t *self); + + /** @brief Calculate the forward pass and write the result to the result tensor. + * + * @param self The layer + */ + void (*forward)(ailayer_t *self); + + // Maybe for later purpose + //void (*sizeof_infmem)(struct aifes_layer_t *, void *); + //void (*set_infmem)(struct aifes_layer_t *, void *); + + // ------------------ Used for training only: ----------------------- + + aitensor_t deltas; /**< The result of the backward function is stored here. */ + + /** @name Training memory API + * @brief Makes the memory of the trainable params, the gradients and optimizer stuff accessible. + * + * This is used, for example, for the optimiser or debugging purposes. + */ + ///@{ + uint8_t trainable_params_count; /**< Number of trainable parameter tensors. */ + aitensor_t **trainable_params; /**< Array of tensor pointers with length trainable_params_count. */ + aitensor_t **gradients; /**< Array of tensor pointers with length trainable_params_count. */ + void **optimem; /**< Array of memory pointers with length trainable_params_count. */ + ///@} + + /** @brief Calculate the backward pass and write the result to the deltas tensor. + * + * @param self The layer + */ + void (*backward)(ailayer_t *self); + + /** @name Parameter memory + * @brief Calculate the size and set the memory for the parameter. + * + * This memory (for example for weights, bias, ...) will last through all the lifetime of the model.\n + * This is only intended for training when no initial weights are available. If the parameters are already known, + * set the parameter directly to the layer. + */ + ///@{ + uint32_t (*sizeof_paramem)(const ailayer_t *self); /**< Size of required memory (in bytes). */ + void (*set_paramem)(ailayer_t *self, void* memory_ptr); /**< Set and distribute the memory block internally. */ + ///@} + + /** @name Training memory + * @brief Calculate the size and set the working memory for the training + * + * This memory (for example for gradients, momentums, ...) is needed during the whole training process. + * If the training is finished, it can be deleted. + */ + ///@{ + uint32_t (*sizeof_trainmem)(const ailayer_t *self); /**< Size of required memory (in bytes). */ + void (*set_trainmem)(ailayer_t *self, void* memory_ptr); /**< Set and distribute the memory block internally. */ + ///@} +}; + + +/** @brief AIfES loss interface. +* +* \image html ailoss.png width=300px +* +* The interface contains the necessary functions and parameters for loss and error calculation on the target data. +* (Refer to aifes_core.h for a structural overview) +*/ +struct ailoss { + const aicore_losstype_t *loss_type; /**< Type of the loss (for example ailoss_mse_type) */ + void *loss_configuration; /**< Loss specific configurations (back-link from abstract loss class to implementation) */ + + ailayer_t connection_layer; /**< Dummy layer for docking to the layer structure */ + + /** @brief Calculate the loss / cost for the model on the given targets. + * + * @param self The layer + * @param target_data Tensor containing the target data / labels + * @param result Scalar in which the output can be written (aiscalar of same type as layer type). + */ + void (*calc_loss)(ailoss_t *self, const aitensor_t *target_data, void *result); + + /** @brief Calculate the error on the target data and write it to the deltas tensor of connection layer + * + * @param self The layer + * @param target_data Tensor containing the target data / labels + */ + void (*calc_delta)(ailoss_t *self, const aitensor_t *target_data); +}; + + +/** @brief AIfES optimizer interface. +* +* \image html aiopti.png width=300px +* +* The interface contains the necessary functions and parameters for parameter optimizers in backpropagation training. +* (Refer to aifes_core.h for a structural overview)\n +* Optimizers are responsible for updating the trainable parameters of the model with the gradients +* calculated in the backward pass. +* +* The call order of the functions is:\n +* \code{.c} +* allocate memory of size sizeof_optimem() +* init_optimem() +* +* for batch in dataset +* for each trainable parameter tensor in the model +* zero_gradients() +* endfor +* +* forward_model(batch) +* backward_model(batch) +* +* begin_step() +* for each trainable parameter tensor in the model +* update_params() +* endfor +* end_step() +* endfor +* \endcode +*/ +struct aiopti { + const aicore_optitype_t *optimizer_type; /**< Type of the optimizer (for example aiopti_sgd_type) */ + void *optimizer_configuration; /**< Optimizer specific configurations (back-link from abstract aiopti class to implementation) */ + const aimath_dtype_t *dtype; /**< The data-type of the parameter that the optimizer can optimize and the learning rate */ + + void *learning_rate; /**< The learning rate configures the training speed. The learning rate is an aiscalar_t value of given dtype. */ + + /** @brief Calculates the optimization memory size for a trainable parameter tensor. + * + * @param self The layer + * @param params The trainable parameter tensor + */ + uint32_t (*sizeof_optimem)(aiopti_t *self, const aitensor_t *params); + + /** @brief Initialize the optimization memory for a trainable parameter tensor. + * + * @param self The layer + * @param params The trainable parameter tensor + * @param gradients The associated gradients tensor + * @param optimem The associated optimization memory to initialize + */ + void (*init_optimem)(aiopti_t *self, const aitensor_t *params, const aitensor_t *gradients, void *optimem); + + /** @brief Set the gradient tensor to zero. + * + * @param self The layer + * @param params The gradient tensor + */ + void (*zero_gradients)(aiopti_t *self, aitensor_t *gradients); + + /** @brief Called in the beginning of every model optimization step for parameter initialization. + * + * @param self The layer + */ + void (*begin_step)(aiopti_t *self); + + /** @brief Performs an optimization step on the given tensor. + * + * @param self The layer + * @param params The trainable parameter tensor + * @param gradients The associated gradients tensor + * @param optimem The associated optimization memory to initialize + */ + void (*update_params)(aiopti_t *self, aitensor_t *params, const aitensor_t *gradients, void *optimem); + + /** @brief Called in the end of every model optimization step + * + * @param self The layer + */ + void (*end_step)(aiopti_t *self); +}; + + +#endif // AIFES_CORE diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/core/aifes_math.h b/examples/2_MNIST_on_PC/1_MNIST_train/src/core/aifes_math.h new file mode 100644 index 0000000..9580f47 --- /dev/null +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/core/aifes_math.h @@ -0,0 +1,107 @@ +/** + * \file core/aifes_math.h + * \internal + * \date 20.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief AIfES 2 math interface + * + * This interface describes the basic structures to perform data type independent math in AIfES. +*/ + +#ifndef AIFES_MATH +#define AIFES_MATH + +#include +#include +#include +#include + + +#define SHAPE_CHECK /** +#include +#include +#include + +#include "aifes.h" + +#include "MNIST_training_data.h" +#include "MNIST_training_data_label.h" +#include "MNIST_test_data.h" +#include "MNIST_test_data_label.h" + +#define INPUTS 784 +#define NEURONS_1 10 +#define NEURONS_2 10 +#define OUTPUTS 10 +#define NUM_TRAIN_DATA 30000 +#define NUM_TEST_DATA 10000 + + +void predict_model_acc(float *model_acc, int num_data, int num_output_neurons, float *data_label, float *model_pred){ + /* + Calculates the accuracy of the neural network + */ + + float argmax_target_acc; + uint8_t argmax_target_index; + float argmax_pred_acc; + uint8_t argmax_pred_index; + *model_acc = 0; + + for(int i=0;i argmax_target_acc) { + argmax_target_acc = *((data_label+i*num_output_neurons)+j); + argmax_target_index = j; + } + if(*((model_pred+i*num_output_neurons)+j) > argmax_pred_acc) { + argmax_pred_acc = *((model_pred+i*num_output_neurons)+j); + argmax_pred_index = j; + } + } + if(argmax_target_index == argmax_pred_index){ + (*model_acc)++; + } + } + *model_acc = (*model_acc/num_data)*100; + +} + + +int main(int argc, char *argv[]) +{ + // loop variables + uint32_t i; + uint32_t j; + uint32_t k; + + float training_acc; + float test_acc; + + + uint16_t input_shape[] = {NUM_TRAIN_DATA, INPUTS}; // Definition of the input shape + aitensor_t input_tensor; // Creation of the input AIfES tensor + input_tensor.dtype = aif32; // Definition of the used data type, here float with 32 bits, different ones are available + input_tensor.dim = 2; // Dimensions of the tensor, here 2 dimensions, as specified at input_shape + input_tensor.shape = input_shape; // Set the shape of the input_tensor + input_tensor.data = MNIST_training_data; // Assign the input_data array to the tensor. It expects a pointer to the array where the data is stored + + + uint16_t input_shape_test_data[] = {NUM_TEST_DATA, INPUTS}; // Definition of the input shape + aitensor_t input_tensor_test_data; // Creation of the input AIfES tensor + input_tensor_test_data.dtype = aif32; // Definition of the used data type, here float with 32 bits, different ones are available + input_tensor_test_data.dim = 2; // Dimensions of the tensor, here 2 dimensions, as specified at input_shape + input_tensor_test_data.shape = input_shape_test_data; // Set the shape of the input_tensor + input_tensor_test_data.data = MNIST_test_data; // Assign the input_data array to the tensor. It expects a pointer to the array where the data is stored + + + uint16_t target_shape[] = {NUM_TRAIN_DATA, OUTPUTS}; // Definition of the output shape + aitensor_t target_tensor; // Creation of the input AIfES tensor + target_tensor.dtype = aif32; // Definition of the used data type, here float with 32 bits, different ones are available + target_tensor.dim = 2; // Dimensions of the tensor, here 2 dimensions, as specified at input_shape + target_tensor.shape = target_shape; // Set the shape of the input_tensor + target_tensor.data = MNIST_training_data_label; // Assign the target_data array to the tensor. It expects a pointer to the array where the data is stored + + + uint16_t target_shape_test_data[] = {NUM_TEST_DATA, OUTPUTS}; // Definition of the output shape + aitensor_t target_tensor_test_data; // Creation of the input AIfES tensor + target_tensor_test_data.dtype = aif32; // Definition of the used data type, here float with 32 bits, different ones are available + target_tensor_test_data.dim = 2; // Dimensions of the tensor, here 2 dimensions, as specified at input_shape + target_tensor_test_data.shape = target_shape_test_data; // Set the shape of the input_tensor + target_tensor_test_data.data = MNIST_test_data_label; // Assign the target_data array to the tensor. It expects a pointer to the array where the data is stored + + // Tensor for the output data (result after training). + // Same configuration as for the target tensor + float output_data[NUM_TRAIN_DATA][OUTPUTS]; + uint16_t output_shape[] = {NUM_TRAIN_DATA, OUTPUTS}; + aitensor_t output_tensor; + output_tensor.dtype = aif32; + output_tensor.dim = 2; + output_tensor.shape = output_shape; + output_tensor.data = output_data; + + // Tensor for the output test data (result of testing). + // Same configuration as for the target tensor + float output_test_data[NUM_TEST_DATA][OUTPUTS]; + uint16_t output_shape_test_data[] = {NUM_TEST_DATA, OUTPUTS}; + aitensor_t output_tensor_test_data; + output_tensor_test_data.dtype = aif32; + output_tensor_test_data.dim = 2; + output_tensor_test_data.shape = output_shape_test_data; + output_tensor_test_data.data = output_test_data; + + // ---------------------------------- Layer definition --------------------------------------- + + // Input layer + uint16_t input_layer_shape[] = {1, INPUTS}; // Definition of the input layer shape (Must fit to the input tensor) + ailayer_input_t input_layer; // Creation of the AIfES input layer + input_layer.input_dim = 2; // Definition of the input dimension (Must fit to the input tensor) + input_layer.input_shape = input_layer_shape; // Handover of the input layer shape + + // Dense layer (hidden layer) + ailayer_dense_t dense_layer_1; // Creation of the AIfES hidden dense layer + dense_layer_1.neurons = NEURONS_1; // Number of neurons + ailayer_relu_t relu_layer_1; // Relu activation function + + // Dense layer (hidden layer_2) + ailayer_dense_t dense_layer_2; // Creation of the AIfES hidden dense layer + dense_layer_2.neurons = NEURONS_2; // Number of neurons + ailayer_relu_t relu_layer_2; // Relu activation function + + // Output dense layer + ailayer_dense_t dense_layer_3; // Creation of the AIfES ouput dense layer + dense_layer_3.neurons = OUTPUTS; // Number of neurons + ailayer_softmax_t softmax_layer_3; // Softmax activation function + + ailoss_crossentropy_t crossentropy_loss; // Loss: Crossentropy + + // --------------------------- Define the structure of the model ---------------------------- + + aimodel_t model; // AIfES model + ailayer_t *x; // Layer object from AIfES, contains the layers + + // Passing the layers to the AIfES model + model.input_layer = ailayer_input_f32_default(&input_layer); + x = ailayer_dense_f32_default(&dense_layer_1, model.input_layer); + x = ailayer_relu_f32_default(&relu_layer_1, x); + x = ailayer_dense_f32_default(&dense_layer_2, x); + x = ailayer_relu_f32_default(&relu_layer_2, x); + x = ailayer_dense_f32_default(&dense_layer_3, x); + x = ailayer_softmax_f32_default(&softmax_layer_3, x); + model.output_layer = x; + + // Add the loss to the AIfES model + model.loss = ailoss_crossentropy_f32_default(&crossentropy_loss, model.output_layer); + + aialgo_compile_model(&model); // Compile the AIfES model + + // ------------------------------- Allocate memory for the parameters of the model ------------------------------ + uint32_t parameter_memory_size = aialgo_sizeof_parameter_memory(&model); + printf("Required memory for parameter (Weights, Bias, ...):"); + printf("%d",parameter_memory_size); + printf("Byte\n"); + + void *parameter_memory = malloc(parameter_memory_size); + + // Distribute the memory to the trainable parameters of the model + aialgo_distribute_parameter_memory(&model, parameter_memory, parameter_memory_size); + + // ------------------------------- Initialize the parameters ------------------------------ + + aimath_f32_default_init_glorot_uniform(&dense_layer_1.weights); + aimath_f32_default_init_zeros(&dense_layer_1.bias); + aimath_f32_default_init_glorot_uniform(&dense_layer_2.weights); + aimath_f32_default_init_zeros(&dense_layer_2.bias); + aimath_f32_default_init_glorot_uniform(&dense_layer_3.weights); + aimath_f32_default_init_zeros(&dense_layer_3.bias); + + // -------------------------------- Define the optimizer for training --------------------- + + aiopti_t *optimizer; // Object for the optimizer + + //ADAM optimizer + aiopti_adam_f32_t adam_opti; + adam_opti.learning_rate = 0.001f; + adam_opti.beta1 = 0.9f; + adam_opti.beta2 = 0.999f; + adam_opti.eps = 1e-7; + + // Choose the optimizer + optimizer = aiopti_adam_f32_default(&adam_opti); + + // -------------------------------- Allocate and schedule the working memory for training --------- + + uint32_t memory_size = aialgo_sizeof_training_memory(&model, optimizer); + printf("Required memory for the training (Intermediate results, gradients, optimization memory): %d Byte\n", memory_size); + + void *memory_ptr = malloc(memory_size); + + // Schedule the memory over the model + aialgo_schedule_training_memory(&model, optimizer, memory_ptr, memory_size); + + // Initialize the AIfES model + aialgo_init_model_for_training(&model, optimizer); + + // --------------------------------- Print the result before training ---------------------------------- + + // Do the inference before training + aialgo_inference_model(&model, &input_tensor, &output_tensor); + + printf("\n"); + printf("Before training:\n"); + printf("Results:\n"); + + for (i = 0; i < 10; i++) { + for(k = 0; k < 2; k++) { + if(k==0){ + printf("real output %d:\n",i); + } + else { + printf("calculated output %d:\n",i); + } + for(j = 0; j < 10; j++) { + if(k==0){ + printf("%f",MNIST_training_data_label[i][j]); + printf("\t"); + } + else { + printf("%f",output_data[i][j]); + printf("\t"); + } + } + } + printf("\n\n"); + } + + predict_model_acc(&training_acc, NUM_TRAIN_DATA, OUTPUTS, &MNIST_training_data_label, &output_data); + printf("Model accuracy before training: %.3f%%\n\n", training_acc); + + // ------------------------------------- Run the training ------------------------------------ + + float loss_training; + uint32_t batch_size = 200; + uint16_t epochs = 50; + uint16_t print_interval = 10; + + printf("\n"); + printf("Start training\n"); + for(i = 0; i < epochs; i++) + { + // One epoch of training. Iterates through the whole data once + aialgo_train_model(&model, &input_tensor, &target_tensor, optimizer, batch_size); + + // Calculate and print loss every print_interval epochs + if(i % print_interval == 0) + { + aialgo_calc_loss_model_f32(&model, &input_tensor, &target_tensor, &loss_training); + + //reduction / normalization of the loss + loss_training = loss_training / (OUTPUTS * NUM_TRAIN_DATA); + + printf("Epoch: "); + printf("%d",i); + printf(" Loss: "); + printf("%f\n",loss_training); + + } + } + printf("Finished training\n\n"); + + // ----------------------------------------- Evaluate the trained model -------------------------- + + + float loss_test; + + // Do the inference after training + aialgo_inference_model(&model, &input_tensor_test_data, &output_tensor_test_data); + + + printf("After training:\n"); + printf("Results:\n"); + + for (i = 0; i < 10; i++) { + for(k = 0; k < 2; k++) { + if(k==0){ + printf("real output %d:\n",i); + } + else { + printf("calculated output %d:\n",i); + } + for(j = 0; j < 10; j++) { + if(k==0){ + printf("%f",MNIST_test_data_label[i][j]); + printf("\t"); + } + else { + printf("%f",output_test_data[i][j]); + printf("\t"); + } + } + } + printf("\n"); + printf("\n"); + } + + predict_model_acc(&test_acc, NUM_TEST_DATA, OUTPUTS, &MNIST_test_data_label, &output_test_data); + printf("Model accuracy after training: %.3f%%\n\n", test_acc); + + aialgo_calc_loss_model_f32(&model, &input_tensor_test_data, &target_tensor_test_data, &loss_test); + + //reduction / normalization of the loss + loss_test = loss_test / (OUTPUTS * NUM_TRAIN_DATA); + printf("Loss: %f\n",loss_test); + + + free(parameter_memory); + free(memory_ptr); + + + system("pause"); + + return 0; +} diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/.gitignore b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/.gitignore new file mode 100644 index 0000000..daedb8a --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/.gitignore @@ -0,0 +1,5 @@ +bin/ +obj/ +/src/MNIST_test_data.h +/src/MNIST_test_data_label.h +/src/MNIST_weights.h \ No newline at end of file diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/LICENSE b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/LICENSE new file mode 100644 index 0000000..f288702 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/MNIST_inference_keras.cbp b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/MNIST_inference_keras.cbp new file mode 100644 index 0000000..45acb64 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/MNIST_inference_keras.cbp @@ -0,0 +1,187 @@ + + + + + + diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/MNIST_inference_keras.depend b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/MNIST_inference_keras.depend new file mode 100644 index 0000000..561213b --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/MNIST_inference_keras.depend @@ -0,0 +1,936 @@ +# depslib dependency file v1.0 +1631882463 source:c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\base\aialgo\aialgo_sequential_inference.c + "basic/base/aialgo/aialgo_sequential_inference.h" + "basic/default/aimath/aimath_f32_default.h" + + + + +1628682756 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\base\aialgo\aialgo_sequential_inference.h + "core/aifes_core.h" + "core/aifes_math.h" + "basic/base/aimath/aimath_basic.h" + "basic/default/ailayer/ailayer_dense_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\core\aifes_core.h + "aifes_math.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\core\aifes_math.h + + + + + +1628682756 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\base\aimath\aimath_basic.h + "core/aifes_math.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\default\ailayer\ailayer_dense_default.h + "basic/base/ailayer/ailayer_dense.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_dense.h + "core/aifes_core.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\default\aimath\aimath_f32_default.h + + + + + "basic/base/aimath/aimath_f32.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\base\aimath\aimath_f32.h + "core/aifes_math.h" + "basic/base/aimath/aimath_basic.h" + +1632130252 source:c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\base\aialgo\aialgo_sequential_training.c + "basic/base/aialgo/aialgo_sequential_training.h" + "basic/base/aialgo/aialgo_sequential_inference.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\base\aialgo\aialgo_sequential_training.h + "core/aifes_core.h" + "core/aifes_math.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_dense.c + "basic/base/ailayer/ailayer_dense.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_elu.c + "basic/base/ailayer/ailayer_elu.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_elu.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_input.c + "basic/base/ailayer/ailayer_input.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_input.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_leaky_relu.c + "basic/base/ailayer/ailayer_leaky_relu.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_leaky_relu.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_relu.c + "basic/base/ailayer/ailayer_relu.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_relu.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_sigmoid.c + "basic/base/ailayer/ailayer_sigmoid.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_sigmoid.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_softmax.c + "basic/base/ailayer/ailayer_softmax.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_softmax.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_softsign.c + "basic/base/ailayer/ailayer_softsign.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_softsign.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_tanh.c + "basic/base/ailayer/ailayer_tanh.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_tanh.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_template.c + "basic/base/ailayer/ailayer_template.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_template.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\base\ailoss\ailoss_crossentropy.c + "basic/base/ailoss/ailoss_crossentropy.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\base\ailoss\ailoss_crossentropy.h + "core/aifes_core.h" + "basic/base/ailayer/ailayer_softmax.h" + "basic/base/ailayer/ailayer_sigmoid.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\base\ailoss\ailoss_mse.c + "basic/base/ailoss/ailoss_mse.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\base\ailoss\ailoss_mse.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\base\aimath\aimath_basic.c + "basic/base/aimath/aimath_basic.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\base\aimath\aimath_f32.c + "basic/base/aimath/aimath_f32.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\base\aiopti\aiopti_adam.c + "basic/base/aiopti/aiopti_adam.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\base\aiopti\aiopti_adam.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\base\aiopti\aiopti_sgd.c + "basic/base/aiopti/aiopti_sgd.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\base\aiopti\aiopti_sgd.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\cmsis\ailayer\ailayer_dense_cmsis.c + "basic/default/ailayer/ailayer_dense_default.h" + "../../../aifes.h" + "basic/cmsis/ailayer/ailayer_dense_cmsis.h" + "basic/cmsis/aimath/aimath_f32_cmsis.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\aifes.h + "core/aifes_math.h" + "core/aifes_core.h" + "basic/base/aimath/aimath_f32.h" + "basic/base/aimath/aimath_basic.h" + "basic/base/ailayer/ailayer_dense.h" + "basic/base/ailayer/ailayer_input.h" + "basic/base/ailayer/ailayer_relu.h" + "basic/base/ailayer/ailayer_leaky_relu.h" + "basic/base/ailayer/ailayer_elu.h" + "basic/base/ailayer/ailayer_sigmoid.h" + "basic/base/ailayer/ailayer_tanh.h" + "basic/base/ailayer/ailayer_softmax.h" + "basic/base/ailayer/ailayer_softsign.h" + "basic/base/ailoss/ailoss_mse.h" + "basic/base/ailoss/ailoss_crossentropy.h" + "basic/base/aiopti/aiopti_sgd.h" + "basic/base/aiopti/aiopti_adam.h" + "basic/default/aimath/aimath_f32_default.h" + "basic/default/ailayer/ailayer_dense_default.h" + "basic/default/ailayer/ailayer_input_default.h" + "basic/default/ailayer/ailayer_relu_default.h" + "basic/default/ailayer/ailayer_leaky_relu_default.h" + "basic/default/ailayer/ailayer_elu_default.h" + "basic/default/ailayer/ailayer_sigmoid_default.h" + "basic/default/ailayer/ailayer_tanh_default.h" + "basic/default/ailayer/ailayer_softmax_default.h" + "basic/default/ailayer/ailayer_softsign_default.h" + "basic/default/ailoss/ailoss_mse_default.h" + "basic/default/ailoss/ailoss_crossentropy_default.h" + "basic/default/aiopti/aiopti_sgd_default.h" + "basic/default/aiopti/aiopti_adam_default.h" + "basic/cmsis/aimath/aimath_f32_cmsis.h" + "basic/cmsis/ailayer/ailayer_dense_cmsis.h" + "basic/base/aialgo/aialgo_sequential_inference.h" + "basic/base/aialgo/aialgo_sequential_training.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\default\ailayer\ailayer_input_default.h + "basic/base/ailayer/ailayer_input.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\default\ailayer\ailayer_relu_default.h + "basic/base/ailayer/ailayer_relu.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\default\ailayer\ailayer_leaky_relu_default.h + "basic/base/ailayer/ailayer_leaky_relu.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\default\ailayer\ailayer_elu_default.h + "basic/base/ailayer/ailayer_elu.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\default\ailayer\ailayer_sigmoid_default.h + "basic/base/ailayer/ailayer_sigmoid.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\default\ailayer\ailayer_tanh_default.h + "basic/base/ailayer/ailayer_tanh.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\default\ailayer\ailayer_softmax_default.h + "basic/base/ailayer/ailayer_softmax.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\default\ailayer\ailayer_softsign_default.h + "basic/base/ailayer/ailayer_softsign.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\default\ailoss\ailoss_mse_default.h + "basic/base/ailoss/ailoss_mse.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\default\ailoss\ailoss_crossentropy_default.h + "basic/base/ailoss/ailoss_crossentropy.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\default\aiopti\aiopti_sgd_default.h + "basic/base/aiopti/aiopti_sgd.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\default\aiopti\aiopti_adam_default.h + "basic/base/aiopti/aiopti_adam.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\cmsis\aimath\aimath_f32_cmsis.h + "basic/base/aimath/aimath_f32.h" + "../../../aifes.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\cmsis\ailayer\ailayer_dense_cmsis.h + "../../../aifes.h" + "basic/base/ailayer/ailayer_dense.h" + "basic/cmsis/aimath/aimath_f32_cmsis.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\cmsis\aimath\aimath_f32_cmsis.c + "basic/cmsis/aimath/aimath_f32_cmsis.h" + "../../../aifes.h" + "arm_math.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\default\ailayer\ailayer_dense_default.c + "basic/default/ailayer/ailayer_dense_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\default\ailayer\ailayer_elu_default.c + "basic/default/ailayer/ailayer_elu_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\default\ailayer\ailayer_input_default.c + "basic/default/ailayer/ailayer_input_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\default\ailayer\ailayer_leaky_relu_default.c + "basic/default/ailayer/ailayer_leaky_relu_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\default\ailayer\ailayer_relu_default.c + "basic/default/ailayer/ailayer_relu_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\default\ailayer\ailayer_sigmoid_default.c + "basic/default/ailayer/ailayer_sigmoid_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\default\ailayer\ailayer_softmax_default.c + "basic/default/ailayer/ailayer_softmax_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\default\ailayer\ailayer_softsign_default.c + "basic/default/ailayer/ailayer_softsign_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\default\ailayer\ailayer_tanh_default.c + "basic/default/ailayer/ailayer_tanh_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\default\ailoss\ailoss_crossentropy_default.c + "basic/default/ailoss/ailoss_crossentropy_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\default\ailoss\ailoss_mse_default.c + "basic/default/ailoss/ailoss_mse_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\default\aimath\aimath_f32_default.c + "basic/default/aimath/aimath_f32_default.h" + + +1628682756 source:c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\default\aiopti\aiopti_adam_default.c + "basic/default/aiopti/aiopti_adam_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\basic\default\aiopti\aiopti_sgd_default.c + "basic/default/aiopti/aiopti_sgd_default.h" + +1632293072 source:c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\main.c + + + + + "aifes.h" + "MNIST_test_data.h" + "MNIST_test_data_label.h" + "MNIST_weights.h" + +1632137043 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\mnist_training_data.h + +1632137043 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\mnist_training_data_label.h + +1632233468 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\mnist_test_data.h + +1632233324 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\mnist_test_data_label.h + +1632293505 c:\users\dk100\documents\aifes\mnist_inference_keras\aifes_for_arduino\src\mnist_weights.h + +1631882463 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\base\aialgo\aialgo_sequential_inference.c + "basic/base/aialgo/aialgo_sequential_inference.h" + "basic/default/aimath/aimath_f32_default.h" + + + + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\base\aialgo\aialgo_sequential_inference.h + "core/aifes_core.h" + "core/aifes_math.h" + "basic/base/aimath/aimath_basic.h" + "basic/default/ailayer/ailayer_dense_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\core\aifes_core.h + "aifes_math.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\core\aifes_math.h + + + + + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\base\aimath\aimath_basic.h + "core/aifes_math.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\default\ailayer\ailayer_dense_default.h + "basic/base/ailayer/ailayer_dense.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\base\ailayer\ailayer_dense.h + "core/aifes_core.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\default\aimath\aimath_f32_default.h + + + + + "basic/base/aimath/aimath_f32.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\base\aimath\aimath_f32.h + "core/aifes_math.h" + "basic/base/aimath/aimath_basic.h" + +1632130252 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\base\aialgo\aialgo_sequential_training.c + "basic/base/aialgo/aialgo_sequential_training.h" + "basic/base/aialgo/aialgo_sequential_inference.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\base\aialgo\aialgo_sequential_training.h + "core/aifes_core.h" + "core/aifes_math.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\base\ailayer\ailayer_dense.c + "basic/base/ailayer/ailayer_dense.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\base\ailayer\ailayer_elu.c + "basic/base/ailayer/ailayer_elu.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\base\ailayer\ailayer_elu.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\base\ailayer\ailayer_input.c + "basic/base/ailayer/ailayer_input.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\base\ailayer\ailayer_input.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\base\ailayer\ailayer_leaky_relu.c + "basic/base/ailayer/ailayer_leaky_relu.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\base\ailayer\ailayer_leaky_relu.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\base\ailayer\ailayer_relu.c + "basic/base/ailayer/ailayer_relu.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\base\ailayer\ailayer_relu.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\base\ailayer\ailayer_sigmoid.c + "basic/base/ailayer/ailayer_sigmoid.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\base\ailayer\ailayer_sigmoid.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\base\ailayer\ailayer_softmax.c + "basic/base/ailayer/ailayer_softmax.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\base\ailayer\ailayer_softmax.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\base\ailayer\ailayer_softsign.c + "basic/base/ailayer/ailayer_softsign.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\base\ailayer\ailayer_softsign.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\base\ailayer\ailayer_tanh.c + "basic/base/ailayer/ailayer_tanh.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\base\ailayer\ailayer_tanh.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\base\ailayer\ailayer_template.c + "basic/base/ailayer/ailayer_template.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\base\ailayer\ailayer_template.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\base\ailoss\ailoss_crossentropy.c + "basic/base/ailoss/ailoss_crossentropy.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\base\ailoss\ailoss_crossentropy.h + "core/aifes_core.h" + "basic/base/ailayer/ailayer_softmax.h" + "basic/base/ailayer/ailayer_sigmoid.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\base\ailoss\ailoss_mse.c + "basic/base/ailoss/ailoss_mse.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\base\ailoss\ailoss_mse.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\base\aimath\aimath_basic.c + "basic/base/aimath/aimath_basic.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\base\aimath\aimath_f32.c + "basic/base/aimath/aimath_f32.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\base\aiopti\aiopti_adam.c + "basic/base/aiopti/aiopti_adam.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\base\aiopti\aiopti_adam.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\base\aiopti\aiopti_sgd.c + "basic/base/aiopti/aiopti_sgd.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\base\aiopti\aiopti_sgd.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\cmsis\ailayer\ailayer_dense_cmsis.c + "basic/default/ailayer/ailayer_dense_default.h" + "../../../aifes.h" + "basic/cmsis/ailayer/ailayer_dense_cmsis.h" + "basic/cmsis/aimath/aimath_f32_cmsis.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\aifes.h + "core/aifes_math.h" + "core/aifes_core.h" + "basic/base/aimath/aimath_f32.h" + "basic/base/aimath/aimath_basic.h" + "basic/base/ailayer/ailayer_dense.h" + "basic/base/ailayer/ailayer_input.h" + "basic/base/ailayer/ailayer_relu.h" + "basic/base/ailayer/ailayer_leaky_relu.h" + "basic/base/ailayer/ailayer_elu.h" + "basic/base/ailayer/ailayer_sigmoid.h" + "basic/base/ailayer/ailayer_tanh.h" + "basic/base/ailayer/ailayer_softmax.h" + "basic/base/ailayer/ailayer_softsign.h" + "basic/base/ailoss/ailoss_mse.h" + "basic/base/ailoss/ailoss_crossentropy.h" + "basic/base/aiopti/aiopti_sgd.h" + "basic/base/aiopti/aiopti_adam.h" + "basic/default/aimath/aimath_f32_default.h" + "basic/default/ailayer/ailayer_dense_default.h" + "basic/default/ailayer/ailayer_input_default.h" + "basic/default/ailayer/ailayer_relu_default.h" + "basic/default/ailayer/ailayer_leaky_relu_default.h" + "basic/default/ailayer/ailayer_elu_default.h" + "basic/default/ailayer/ailayer_sigmoid_default.h" + "basic/default/ailayer/ailayer_tanh_default.h" + "basic/default/ailayer/ailayer_softmax_default.h" + "basic/default/ailayer/ailayer_softsign_default.h" + "basic/default/ailoss/ailoss_mse_default.h" + "basic/default/ailoss/ailoss_crossentropy_default.h" + "basic/default/aiopti/aiopti_sgd_default.h" + "basic/default/aiopti/aiopti_adam_default.h" + "basic/cmsis/aimath/aimath_f32_cmsis.h" + "basic/cmsis/ailayer/ailayer_dense_cmsis.h" + "basic/base/aialgo/aialgo_sequential_inference.h" + "basic/base/aialgo/aialgo_sequential_training.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\default\ailayer\ailayer_input_default.h + "basic/base/ailayer/ailayer_input.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\default\ailayer\ailayer_relu_default.h + "basic/base/ailayer/ailayer_relu.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\default\ailayer\ailayer_leaky_relu_default.h + "basic/base/ailayer/ailayer_leaky_relu.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\default\ailayer\ailayer_elu_default.h + "basic/base/ailayer/ailayer_elu.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\default\ailayer\ailayer_sigmoid_default.h + "basic/base/ailayer/ailayer_sigmoid.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\default\ailayer\ailayer_tanh_default.h + "basic/base/ailayer/ailayer_tanh.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\default\ailayer\ailayer_softmax_default.h + "basic/base/ailayer/ailayer_softmax.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\default\ailayer\ailayer_softsign_default.h + "basic/base/ailayer/ailayer_softsign.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\default\ailoss\ailoss_mse_default.h + "basic/base/ailoss/ailoss_mse.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\default\ailoss\ailoss_crossentropy_default.h + "basic/base/ailoss/ailoss_crossentropy.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\default\aiopti\aiopti_sgd_default.h + "basic/base/aiopti/aiopti_sgd.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\default\aiopti\aiopti_adam_default.h + "basic/base/aiopti/aiopti_adam.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\cmsis\aimath\aimath_f32_cmsis.h + "basic/base/aimath/aimath_f32.h" + "../../../aifes.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\cmsis\ailayer\ailayer_dense_cmsis.h + "../../../aifes.h" + "basic/base/ailayer/ailayer_dense.h" + "basic/cmsis/aimath/aimath_f32_cmsis.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\cmsis\aimath\aimath_f32_cmsis.c + "basic/cmsis/aimath/aimath_f32_cmsis.h" + "../../../aifes.h" + "arm_math.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\default\ailayer\ailayer_dense_default.c + "basic/default/ailayer/ailayer_dense_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\default\ailayer\ailayer_elu_default.c + "basic/default/ailayer/ailayer_elu_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\default\ailayer\ailayer_input_default.c + "basic/default/ailayer/ailayer_input_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\default\ailayer\ailayer_leaky_relu_default.c + "basic/default/ailayer/ailayer_leaky_relu_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\default\ailayer\ailayer_relu_default.c + "basic/default/ailayer/ailayer_relu_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\default\ailayer\ailayer_sigmoid_default.c + "basic/default/ailayer/ailayer_sigmoid_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\default\ailayer\ailayer_softmax_default.c + "basic/default/ailayer/ailayer_softmax_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\default\ailayer\ailayer_softsign_default.c + "basic/default/ailayer/ailayer_softsign_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\default\ailayer\ailayer_tanh_default.c + "basic/default/ailayer/ailayer_tanh_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\default\ailoss\ailoss_crossentropy_default.c + "basic/default/ailoss/ailoss_crossentropy_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\default\ailoss\ailoss_mse_default.c + "basic/default/ailoss/ailoss_mse_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\default\aimath\aimath_f32_default.c + "basic/default/aimath/aimath_f32_default.h" + + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\default\aiopti\aiopti_adam_default.c + "basic/default/aiopti/aiopti_adam_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_inference_keras\src\basic\default\aiopti\aiopti_sgd_default.c + "basic/default/aiopti/aiopti_sgd_default.h" + +1631882463 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\base\aialgo\aialgo_sequential_inference.c + "basic/base/aialgo/aialgo_sequential_inference.h" + "basic/default/aimath/aimath_f32_default.h" + + + + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\base\aialgo\aialgo_sequential_inference.h + "core/aifes_core.h" + "core/aifes_math.h" + "basic/base/aimath/aimath_basic.h" + "basic/default/ailayer/ailayer_dense_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\core\aifes_core.h + "aifes_math.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\core\aifes_math.h + + + + + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\base\aimath\aimath_basic.h + "core/aifes_math.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\default\ailayer\ailayer_dense_default.h + "basic/base/ailayer/ailayer_dense.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\base\ailayer\ailayer_dense.h + "core/aifes_core.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\default\aimath\aimath_f32_default.h + + + + + "basic/base/aimath/aimath_f32.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\base\aimath\aimath_f32.h + "core/aifes_math.h" + "basic/base/aimath/aimath_basic.h" + +1632130252 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\base\aialgo\aialgo_sequential_training.c + "basic/base/aialgo/aialgo_sequential_training.h" + "basic/base/aialgo/aialgo_sequential_inference.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\base\aialgo\aialgo_sequential_training.h + "core/aifes_core.h" + "core/aifes_math.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\base\ailayer\ailayer_dense.c + "basic/base/ailayer/ailayer_dense.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\base\ailayer\ailayer_elu.c + "basic/base/ailayer/ailayer_elu.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\base\ailayer\ailayer_elu.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\base\ailayer\ailayer_input.c + "basic/base/ailayer/ailayer_input.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\base\ailayer\ailayer_input.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\base\ailayer\ailayer_leaky_relu.c + "basic/base/ailayer/ailayer_leaky_relu.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\base\ailayer\ailayer_leaky_relu.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\base\ailayer\ailayer_relu.c + "basic/base/ailayer/ailayer_relu.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\base\ailayer\ailayer_relu.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\base\ailayer\ailayer_sigmoid.c + "basic/base/ailayer/ailayer_sigmoid.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\base\ailayer\ailayer_sigmoid.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\base\ailayer\ailayer_softmax.c + "basic/base/ailayer/ailayer_softmax.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\base\ailayer\ailayer_softmax.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\base\ailayer\ailayer_softsign.c + "basic/base/ailayer/ailayer_softsign.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\base\ailayer\ailayer_softsign.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\base\ailayer\ailayer_tanh.c + "basic/base/ailayer/ailayer_tanh.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\base\ailayer\ailayer_tanh.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\base\ailayer\ailayer_template.c + "basic/base/ailayer/ailayer_template.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\base\ailayer\ailayer_template.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\base\ailoss\ailoss_crossentropy.c + "basic/base/ailoss/ailoss_crossentropy.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\base\ailoss\ailoss_crossentropy.h + "core/aifes_core.h" + "basic/base/ailayer/ailayer_softmax.h" + "basic/base/ailayer/ailayer_sigmoid.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\base\ailoss\ailoss_mse.c + "basic/base/ailoss/ailoss_mse.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\base\ailoss\ailoss_mse.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\base\aimath\aimath_basic.c + "basic/base/aimath/aimath_basic.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\base\aimath\aimath_f32.c + "basic/base/aimath/aimath_f32.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\base\aiopti\aiopti_adam.c + "basic/base/aiopti/aiopti_adam.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\base\aiopti\aiopti_adam.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\base\aiopti\aiopti_sgd.c + "basic/base/aiopti/aiopti_sgd.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\base\aiopti\aiopti_sgd.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\cmsis\ailayer\ailayer_dense_cmsis.c + "basic/default/ailayer/ailayer_dense_default.h" + "../../../aifes.h" + "basic/cmsis/ailayer/ailayer_dense_cmsis.h" + "basic/cmsis/aimath/aimath_f32_cmsis.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\aifes.h + "core/aifes_math.h" + "core/aifes_core.h" + "basic/base/aimath/aimath_f32.h" + "basic/base/aimath/aimath_basic.h" + "basic/base/ailayer/ailayer_dense.h" + "basic/base/ailayer/ailayer_input.h" + "basic/base/ailayer/ailayer_relu.h" + "basic/base/ailayer/ailayer_leaky_relu.h" + "basic/base/ailayer/ailayer_elu.h" + "basic/base/ailayer/ailayer_sigmoid.h" + "basic/base/ailayer/ailayer_tanh.h" + "basic/base/ailayer/ailayer_softmax.h" + "basic/base/ailayer/ailayer_softsign.h" + "basic/base/ailoss/ailoss_mse.h" + "basic/base/ailoss/ailoss_crossentropy.h" + "basic/base/aiopti/aiopti_sgd.h" + "basic/base/aiopti/aiopti_adam.h" + "basic/default/aimath/aimath_f32_default.h" + "basic/default/ailayer/ailayer_dense_default.h" + "basic/default/ailayer/ailayer_input_default.h" + "basic/default/ailayer/ailayer_relu_default.h" + "basic/default/ailayer/ailayer_leaky_relu_default.h" + "basic/default/ailayer/ailayer_elu_default.h" + "basic/default/ailayer/ailayer_sigmoid_default.h" + "basic/default/ailayer/ailayer_tanh_default.h" + "basic/default/ailayer/ailayer_softmax_default.h" + "basic/default/ailayer/ailayer_softsign_default.h" + "basic/default/ailoss/ailoss_mse_default.h" + "basic/default/ailoss/ailoss_crossentropy_default.h" + "basic/default/aiopti/aiopti_sgd_default.h" + "basic/default/aiopti/aiopti_adam_default.h" + "basic/cmsis/aimath/aimath_f32_cmsis.h" + "basic/cmsis/ailayer/ailayer_dense_cmsis.h" + "basic/base/aialgo/aialgo_sequential_inference.h" + "basic/base/aialgo/aialgo_sequential_training.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\default\ailayer\ailayer_input_default.h + "basic/base/ailayer/ailayer_input.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\default\ailayer\ailayer_relu_default.h + "basic/base/ailayer/ailayer_relu.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\default\ailayer\ailayer_leaky_relu_default.h + "basic/base/ailayer/ailayer_leaky_relu.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\default\ailayer\ailayer_elu_default.h + "basic/base/ailayer/ailayer_elu.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\default\ailayer\ailayer_sigmoid_default.h + "basic/base/ailayer/ailayer_sigmoid.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\default\ailayer\ailayer_tanh_default.h + "basic/base/ailayer/ailayer_tanh.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\default\ailayer\ailayer_softmax_default.h + "basic/base/ailayer/ailayer_softmax.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\default\ailayer\ailayer_softsign_default.h + "basic/base/ailayer/ailayer_softsign.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\default\ailoss\ailoss_mse_default.h + "basic/base/ailoss/ailoss_mse.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\default\ailoss\ailoss_crossentropy_default.h + "basic/base/ailoss/ailoss_crossentropy.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\default\aiopti\aiopti_sgd_default.h + "basic/base/aiopti/aiopti_sgd.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\default\aiopti\aiopti_adam_default.h + "basic/base/aiopti/aiopti_adam.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\cmsis\aimath\aimath_f32_cmsis.h + "basic/base/aimath/aimath_f32.h" + "../../../aifes.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\cmsis\ailayer\ailayer_dense_cmsis.h + "../../../aifes.h" + "basic/base/ailayer/ailayer_dense.h" + "basic/cmsis/aimath/aimath_f32_cmsis.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\cmsis\aimath\aimath_f32_cmsis.c + "basic/cmsis/aimath/aimath_f32_cmsis.h" + "../../../aifes.h" + "arm_math.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\default\ailayer\ailayer_dense_default.c + "basic/default/ailayer/ailayer_dense_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\default\ailayer\ailayer_elu_default.c + "basic/default/ailayer/ailayer_elu_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\default\ailayer\ailayer_input_default.c + "basic/default/ailayer/ailayer_input_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\default\ailayer\ailayer_leaky_relu_default.c + "basic/default/ailayer/ailayer_leaky_relu_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\default\ailayer\ailayer_relu_default.c + "basic/default/ailayer/ailayer_relu_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\default\ailayer\ailayer_sigmoid_default.c + "basic/default/ailayer/ailayer_sigmoid_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\default\ailayer\ailayer_softmax_default.c + "basic/default/ailayer/ailayer_softmax_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\default\ailayer\ailayer_softsign_default.c + "basic/default/ailayer/ailayer_softsign_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\default\ailayer\ailayer_tanh_default.c + "basic/default/ailayer/ailayer_tanh_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\default\ailoss\ailoss_crossentropy_default.c + "basic/default/ailoss/ailoss_crossentropy_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\default\ailoss\ailoss_mse_default.c + "basic/default/ailoss/ailoss_mse_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\default\aimath\aimath_f32_default.c + "basic/default/aimath/aimath_f32_default.h" + + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\default\aiopti\aiopti_adam_default.c + "basic/default/aiopti/aiopti_adam_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\2_mnist_inference_keras\src\basic\default\aiopti\aiopti_sgd_default.c + "basic/default/aiopti/aiopti_sgd_default.h" + diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/MNIST_inference_keras.layout b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/MNIST_inference_keras.layout new file mode 100644 index 0000000..1c9fb71 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/MNIST_inference_keras.layout @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/keywords.txt b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/keywords.txt new file mode 100644 index 0000000..6d759a9 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/keywords.txt @@ -0,0 +1,217 @@ +####################################### +# Syntax Coloring Map For AIfES for Arduino +####################################### + +####################################### +# Datatypes (KEYWORD1) +####################################### + +aimodel_t KEYWORD1 +ailayer_t KEYWORD1 +ailoss_t KEYWORD1 +aiopti_t KEYWORD1 + +aicore_layertype_t KEYWORD1 +aicore_losstype_t KEYWORD1 +aicore_optitype_t KEYWORD1 + +ailayer_dense_t KEYWORD1 +ailayer_elu_t KEYWORD1 +ailayer_elu_f32_t KEYWORD1 +ailayer_input_t KEYWORD1 +ailayer_leaky_relu_t KEYWORD1 +ailayer_leaky_relu_f32_t KEYWORD1 +ailayer_relu_t KEYWORD1 +ailayer_sigmoid_t KEYWORD1 +ailayer_softmax_t KEYWORD1 +ailayer_softsign_t KEYWORD1 +ailayer_tanh_t KEYWORD1 + +ailoss_crossentropy_t KEYWORD1 +ailoss_mse_t KEYWORD1 + +aiopti_adam_t KEYWORD1 +aiopti_adam_f32_t KEYWORD1 +aiopti_adam_momentums_t KEYWORD1 +aiopti_sgd_t KEYWORD1 +aiopti_sgd_f32_t KEYWORD1 + +aitensor_t KEYWORD1 +####################################### +# Methods and Functions (KEYWORD2) +####################################### + +aialgo_backward_model KEYWORD2 +aialgo_calc_loss_model_f32 KEYWORD2 +aialgo_compile_model KEYWORD2 +aialgo_distribute_parameter_memory KEYWORD2 +aialgo_forward_model KEYWORD2 +aialgo_inference_model KEYWORD2 +aialgo_init_model_for_training KEYWORD2 +aialgo_print_loss_specs KEYWORD2 +aialgo_print_model_structure KEYWORD2 +aialgo_print_optimizer_specs KEYWORD2 +aialgo_schedule_inference_memory KEYWORD2 +aialgo_schedule_training_memory KEYWORD2 +aialgo_sizeof_inference_memory KEYWORD2 +aialgo_sizeof_parameter_memory KEYWORD2 +aialgo_sizeof_training_memory KEYWORD2 +aialgo_train_model KEYWORD2 +aialgo_update_params_model KEYWORD2 +aialgo_zero_gradients_model KEYWORD2 +ailayer_dense KEYWORD2 +ailayer_dense_backward KEYWORD2 +ailayer_dense_calc_result_shape KEYWORD2 +ailayer_dense_f32_cmsis KEYWORD2 +ailayer_dense_f32_default KEYWORD2 +ailayer_dense_forward KEYWORD2 +ailayer_dense_print_specs KEYWORD2 +ailayer_dense_set_paramem KEYWORD2 +ailayer_dense_set_trainmem KEYWORD2 +ailayer_dense_sizeof_paramem KEYWORD2 +ailayer_dense_sizeof_trainmem KEYWORD2 +ailayer_elu KEYWORD2 +ailayer_elu_backward KEYWORD2 +ailayer_elu_calc_result_shape KEYWORD2 +ailayer_elu_f32_default KEYWORD2 +ailayer_elu_forward KEYWORD2 +ailayer_elu_print_specs KEYWORD2 +ailayer_input KEYWORD2 +ailayer_input_backward KEYWORD2 +ailayer_input_calc_result_shape KEYWORD2 +ailayer_input_f32_default KEYWORD2 +ailayer_input_forward KEYWORD2 +ailayer_input_print_specs KEYWORD2 +ailayer_leaky_relu KEYWORD2 +ailayer_leaky_relu_backward KEYWORD2 +ailayer_leaky_relu_calc_result_shape KEYWORD2 +ailayer_leaky_relu_f32_default KEYWORD2 +ailayer_leaky_relu_forward KEYWORD2 +ailayer_leaky_relu_print_specs KEYWORD2 +ailayer_relu KEYWORD2 +ailayer_relu_backward KEYWORD2 +ailayer_relu_calc_result_shape KEYWORD2 +ailayer_relu_f32_default KEYWORD2 +ailayer_relu_forward KEYWORD2 +ailayer_relu_print_specs KEYWORD2 +ailayer_sigmoid KEYWORD2 +ailayer_sigmoid_backward KEYWORD2 +ailayer_sigmoid_calc_result_shape KEYWORD2 +ailayer_sigmoid_f32_default KEYWORD2 +ailayer_sigmoid_forward KEYWORD2 +ailayer_sigmoid_get_result_bound_f32_default KEYWORD2 +ailayer_sigmoid_print_specs KEYWORD2 +ailayer_softmax KEYWORD2 +ailayer_softmax_calc_result_shape KEYWORD2 +ailayer_softmax_f32_default KEYWORD2 +ailayer_softmax_forward KEYWORD2 +ailayer_softmax_print_specs KEYWORD2 +ailayer_softsign KEYWORD2 +ailayer_softsign_backward KEYWORD2 +ailayer_softsign_calc_result_shape KEYWORD2 +ailayer_softsign_f32_default KEYWORD2 +ailayer_softsign_forward KEYWORD2 +ailayer_softsign_print_specs KEYWORD2 +ailayer_tanh KEYWORD2 +ailayer_tanh_backward KEYWORD2 +ailayer_tanh_calc_result_shape KEYWORD2 +ailayer_tanh_f32_default KEYWORD2 +ailayer_tanh_forward KEYWORD2 +ailayer_tanh_get_result_bound_f32_default KEYWORD2 +ailayer_tanh_print_specs KEYWORD2 +ailayer_template KEYWORD2 +ailayer_template_backward KEYWORD2 +ailayer_template_calc_result_shape KEYWORD2 +ailayer_template_forward KEYWORD2 +ailayer_template_print_specs KEYWORD2 +ailayer_template_set_paramem KEYWORD2 +ailayer_template_set_trainmem KEYWORD2 +ailayer_template_sizeof_paramem KEYWORD2 +ailayer_template_sizeof_trainmem KEYWORD2 +ailoss_crossentropy KEYWORD2 +ailoss_crossentropy_calc_delta KEYWORD2 +ailoss_crossentropy_calc_loss KEYWORD2 +ailoss_crossentropy_dummy_backward KEYWORD2 +ailoss_crossentropy_f32_default KEYWORD2 +ailoss_crossentropy_print_specs KEYWORD2 +ailoss_mse KEYWORD2 +ailoss_mse_calc_delta KEYWORD2 +ailoss_mse_calc_loss KEYWORD2 +ailoss_mse_f32_default KEYWORD2 +ailoss_mse_print_specs KEYWORD2 +aimath_f32_cmsis_linear KEYWORD2 +aimath_f32_cmsis_mat_mul KEYWORD2 +aimath_f32_default_binary_crossentropy KEYWORD2 +aimath_f32_default_categorical_crossentropy KEYWORD2 +aimath_f32_default_categorical_crossentropy_sparse8 KEYWORD2 +aimath_f32_default_copy_tensor KEYWORD2 +aimath_f32_default_d_elu KEYWORD2 +aimath_f32_default_d_leaky_relu KEYWORD2 +aimath_f32_default_d_relu KEYWORD2 +aimath_f32_default_d_sigmoid KEYWORD2 +aimath_f32_default_d_softsign KEYWORD2 +aimath_f32_default_d_tanh KEYWORD2 +aimath_f32_default_divide KEYWORD2 +aimath_f32_default_elu KEYWORD2 +aimath_f32_default_expf_fast KEYWORD2 +aimath_f32_default_init_glorot_uniform KEYWORD2 +aimath_f32_default_init_he_uniform KEYWORD2 +aimath_f32_default_init_zeros KEYWORD2 +aimath_f32_default_leaky_relu KEYWORD2 +aimath_f32_default_linear KEYWORD2 +aimath_f32_default_mat_mul KEYWORD2 +aimath_f32_default_max KEYWORD2 +aimath_f32_default_min KEYWORD2 +aimath_f32_default_multiply KEYWORD2 +aimath_f32_default_norm_squared KEYWORD2 +aimath_f32_default_relu KEYWORD2 +aimath_f32_default_scalar_add KEYWORD2 +aimath_f32_default_scalar_mul KEYWORD2 +aimath_f32_default_sigmoid KEYWORD2 +aimath_f32_default_softmax KEYWORD2 +aimath_f32_default_softsign KEYWORD2 +aimath_f32_default_sqrt KEYWORD2 +aimath_f32_default_sum KEYWORD2 +aimath_f32_default_tanh KEYWORD2 +aimath_f32_default_tensor_add KEYWORD2 +aimath_f32_default_tensor_init_uniform KEYWORD2 +aimath_f32_default_tensor_sub KEYWORD2 +aimath_f32_default_tensor_sub_sparse8 KEYWORD2 +aimath_f32_default_transpose_vector KEYWORD2 +aimath_f32_default_zero_tensor KEYWORD2 +aimath_f32_print_aiscalar KEYWORD2 +aimath_f32_print_aitensor KEYWORD2 +aimath_sizeof_dtype KEYWORD2 +aimath_sizeof_tensor KEYWORD2 +aimath_sizeof_tensor_data KEYWORD2 +aimath_sizeof_tensor_params KEYWORD2 +aimath_tensor_elements KEYWORD2 +aimath_transpose_vector KEYWORD2 +aiopti_adam KEYWORD2 +aiopti_adam_f32_default KEYWORD2 +aiopti_adam_f32_default_begin_step KEYWORD2 +aiopti_adam_f32_default_end_step KEYWORD2 +aiopti_adam_init_optimem KEYWORD2 +aiopti_adam_print_specs KEYWORD2 +aiopti_adam_sizeof_optimem KEYWORD2 +aiopti_adam_update_params KEYWORD2 +aiopti_adam_zero_gradients KEYWORD2 +aiopti_sgd KEYWORD2 +aiopti_sgd_f32_default KEYWORD2 +aiopti_sgd_init_optimem_with_momentum KEYWORD2 +aiopti_sgd_init_optimem_without_momentum KEYWORD2 +aiopti_sgd_print_specs KEYWORD2 +aiopti_sgd_sizeof_optimem_with_momentum KEYWORD2 +aiopti_sgd_sizeof_optimem_without_momentum KEYWORD2 +aiopti_sgd_update_params_with_momentum KEYWORD2 +aiopti_sgd_update_params_without_momentum KEYWORD2 +aiopti_sgd_zero_gradients KEYWORD2 + +print_aiscalar KEYWORD2 +print_aitensor KEYWORD2 + +####################################### +# Constants (LITERAL1) +####################################### + +aif32 LITERAL1 diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/MNIST_to_cc.py b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/MNIST_to_cc.py new file mode 100644 index 0000000..ffbe941 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/MNIST_to_cc.py @@ -0,0 +1,81 @@ +import tensorflow as tf +from tensorflow.keras.utils import to_categorical + + +#Load and preprocess the MNIST data set +(x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data() + +x_train = x_train.astype(float)/255.0 +x_test = x_test.astype(float)/255.0 + +y_train = to_categorical(y_train) +y_test = to_categorical(y_test) + +print(x_train.shape) +print(x_test.shape) + +NUM_TRAINING_DATA = 30000 #Max 60000 +NUM_TEST_DATA = 10000 #Max 10000 + +def generate_train_data(): + ''' + Generates two header files. The first one contains as many training data according to the + number defined above. The second header file contains the labels belonging to the training + data. The training data and labels are both stored in a two-dimensional array. + ''' + + with open("MNIST_training_data.h", "w") as f: + f.write("float MNIST_training_data[" + str(NUM_TRAINING_DATA) + "][784] = {\n") + for i in range(0,NUM_TRAINING_DATA): + if i != 0: + f.write("},\n") + x_train_flatten = x_train[i].flatten() + f.write("{" + str(x_train_flatten[0]) + "f") + for j in range(1,784): + f.write(", " + str(x_train_flatten[j]) + "f") + f.write("}\n};") + + + with open("MNIST_training_data_label.h", "w") as f: + f.write("float MNIST_training_data_label[" + str(NUM_TRAINING_DATA) + "][10] = {\n") + for i in range(0,NUM_TRAINING_DATA): + if i != 0: + f.write("},\n") + f.write("{" + str(y_train[i][0]) + "f") + for j in range(1,10): + f.write(", " + str(y_train[i][j]) + "f") + f.write("}\n};") + + +def generate_test_data(): + ''' + Generates two header files. The first one contains as many test data according to the + number defined above. The second header file contains the labels belonging to the test + data. The test data and labels are both stored in a two-dimensional array. + ''' + + with open("MNIST_test_data.h", "w") as f: + f.write("float MNIST_test_data[" + str(NUM_TEST_DATA) + "][784] = {\n") + for i in range(0,NUM_TEST_DATA): + if i != 0: + f.write("},\n") + x_test_flatten = x_test[i].flatten() + f.write("{" + str(x_test_flatten[0]) + "f") + for j in range(1,784): + f.write(", " + str(x_test_flatten[j]) + "f") + f.write("}\n};") + + + with open("MNIST_test_data_label.h", "w") as f: + f.write("float MNIST_test_data_label[" + str(NUM_TEST_DATA) + "][10] = {\n") + for i in range(0,NUM_TEST_DATA): + if i != 0: + f.write("},\n") + f.write("{" + str(y_test[i][0]) + "f") + for j in range(1,10): + f.write(", " + str(y_test[i][j]) + "f") + f.write("}\n};") + + +# generate_train_data() +generate_test_data() \ No newline at end of file diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/MNIST_weights.py b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/MNIST_weights.py new file mode 100644 index 0000000..aa17b49 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/MNIST_weights.py @@ -0,0 +1,98 @@ +import tensorflow as tf +from tensorflow.keras.utils import to_categorical + +from tensorflow.keras.models import Sequential +from tensorflow.keras.layers import Dense + +import cv2 + + +def train_model(): + ''' + Load and preprocess the MNIST data set and train a DNN. + ''' + + (x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data() + + x_train = x_train.astype(float)/255.0 + x_test = x_test.astype(float)/255.0 + + x_train = x_train.reshape(60000, 784) + x_test = x_test.reshape(10000, 784) + + y_train = to_categorical(y_train, 10) + y_test = to_categorical(y_test, 10) + + print(x_train.shape) + print(x_test.shape) + + + model = Sequential() + model.add(Dense(10, activation='relu', input_shape=(784,))) + model.add(Dense(10, activation='relu')) + model.add(Dense(10, activation='softmax')) + + model.summary() + + model.compile(optimizer="Adam", loss="categorical_crossentropy", metrics=["acc"]) + + model.fit(x_train, y_train, validation_split=0.2, epochs=10, batch_size=100) + + print("Accuracy on test data:") + model.evaluate(x_test, y_test) + + return model + + +def write_weights(model): + ''' + Load weights of the passed neural network. For each layer, the weights and the + bias are saved in a header file as an array. + ''' + + # Get the weights + weights = model.get_weights() + + # Write the weights for AIfES in a .h file + with open("MNIST_weights.h", "w") as f: + for k in range(0,len(weights)): + if (k%2) == 0: + f.write("float weights_data_dense_" + str(int(k/2)+1) + "[] = {") + else: + f.write("float bias_data_dense_" + str(int(k/2)+1) + "[] = {") + + if len(weights[k].shape) == 2: + for i in range(0,weights[k].shape[0]): + for j in range(0,weights[k].shape[1]): + if i == (weights[k].shape[0]-1) and j == (weights[k].shape[1]-1): + f.write(str(weights[k][i, j]) + "f") + else: + f.write(str(weights[k][i, j]) + "f, ") + f.write("};\n") + else: + for i in range(0,weights[k].shape[0]): + if i == (weights[k].shape[0]-1): + f.write(str(weights[k][i]) + "f") + else: + f.write(str(weights[k][i]) + "f, ") + f.write("};\n") + + f.write("\n") + + +def show_MNIST_test_img(i): + ''' + An image of the test data set can be displayed for verification. + ''' + + (_, _), (x_test, y_test) = tf.keras.datasets.mnist.load_data() + x_test = x_test.astype(float)/255.0 + img = cv2.resize(x_test[i], (200, 200)) + print(y_test[i]) + cv2.imshow('image',img) + cv2.waitKey(0) + + +mnist_model = train_model() +write_weights(mnist_model) +# show_MNIST_test_img(18) \ No newline at end of file diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/aifes.h b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/aifes.h new file mode 100644 index 0000000..e7af402 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/aifes.h @@ -0,0 +1,113 @@ +/** + * \file basic/aifes.h + * \internal + * \date 23.02.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Include all headers of the aifes 2 - basic module + * \details + */ + +#ifdef __cplusplus +extern "C" { +#endif + + +// Include AIfES core headers +#include "core/aifes_math.h" +#include "core/aifes_core.h" + +// Include the datatypes +#include "basic/base/aimath/aimath_f32.h" + +// Include basic datatype independent math functions +#include "basic/base/aimath/aimath_basic.h" + +// ---------------------------- Module base implementations ----------------------- +// ("abstract" super "classes". A hardware optimized implementation can "inherit" from these modules) + +// Include the layer base implementations +#include "basic/base/ailayer/ailayer_dense.h" +#include "basic/base/ailayer/ailayer_input.h" +#include "basic/base/ailayer/ailayer_relu.h" +#include "basic/base/ailayer/ailayer_leaky_relu.h" +#include "basic/base/ailayer/ailayer_elu.h" +#include "basic/base/ailayer/ailayer_sigmoid.h" +#include "basic/base/ailayer/ailayer_tanh.h" +#include "basic/base/ailayer/ailayer_softmax.h" +#include "basic/base/ailayer/ailayer_softsign.h" + +// Include the loss base implementations +#include "basic/base/ailoss/ailoss_mse.h" +#include "basic/base/ailoss/ailoss_crossentropy.h" + +// Include the optimizer base implementations +#include "basic/base/aiopti/aiopti_sgd.h" +#include "basic/base/aiopti/aiopti_adam.h" + +// ---------------------------- Module default implementations ----------------------- +// (Fallback functions if no hardware optimized implementation available) + +// Include the math in default implementation +#include "basic/default/aimath/aimath_f32_default.h" + +// Include the layers in default implementation +#include "basic/default/ailayer/ailayer_dense_default.h" +#include "basic/default/ailayer/ailayer_input_default.h" +#include "basic/default/ailayer/ailayer_relu_default.h" +#include "basic/default/ailayer/ailayer_leaky_relu_default.h" +#include "basic/default/ailayer/ailayer_elu_default.h" +#include "basic/default/ailayer/ailayer_sigmoid_default.h" +#include "basic/default/ailayer/ailayer_tanh_default.h" +#include "basic/default/ailayer/ailayer_softmax_default.h" +#include "basic/default/ailayer/ailayer_softsign_default.h" + +// Include the losses in default implementation +#include "basic/default/ailoss/ailoss_mse_default.h" +#include "basic/default/ailoss/ailoss_crossentropy_default.h" + +// Include the optimizers in default implementation +#include "basic/default/aiopti/aiopti_sgd_default.h" +#include "basic/default/aiopti/aiopti_adam_default.h" + +// ---------------------------- CMSIS implementations ----------------------- +// ATTENTION! +// If you want to use CMSIS for ARM controllers, you need to uncomment the define of AIFES_WITH_CMSIS + +//#define AIFES_WITH_CMSIS + +#ifdef AIFES_WITH_CMSIS + +// Include the math in cmsis implementation +#include "basic/cmsis/aimath/aimath_f32_cmsis.h" + +// Include the layers in cmsis implementation +#include "basic/cmsis/ailayer/ailayer_dense_cmsis.h" + +#endif /* AIFES_USE_CMSIS */ + +// ---------------------------- Algorithmic ----------------------- + +// Include the algorithmic +#include "basic/base/aialgo/aialgo_sequential_inference.h" +#include "basic/base/aialgo/aialgo_sequential_training.h" + +#ifdef __cplusplus +} // End extern "C" +#endif diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aialgo/aialgo_sequential_inference.c b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aialgo/aialgo_sequential_inference.c new file mode 100644 index 0000000..b11b867 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aialgo/aialgo_sequential_inference.c @@ -0,0 +1,224 @@ +/** + * \file basic/base/aialgo/aialgo_sequential_inference.c + * \version 2.0alpha + * \date 20.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/aialgo/aialgo_sequential_inference.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +#include +#include +#include + +uint32_t aialgo_sizeof_inference_memory(aimodel_t *model) +{ + uint16_t i; + uint32_t memory = 0, max_memory = 0; + ailayer_t *layer_ptr = model->input_layer; + + // Calculate result shapes and max amount of memory + for(i = 0; i < model->layer_count; i++) + { + layer_ptr->calc_result_shape(layer_ptr); + memory = aimath_sizeof_tensor_data(&(layer_ptr->result)); + if(memory > max_memory) max_memory = memory; + + layer_ptr = layer_ptr->output_layer; + } + + return 2 * max_memory; // Input and output buffer +} + +uint32_t aialgo_sizeof_parameter_memory(aimodel_t *model) +{ + int i; + ailayer_t *layer_ptr = model->input_layer; + uint32_t memory = 0; + + for(i = 0; i < model->layer_count; i++) + { + layer_ptr->calc_result_shape(layer_ptr); + + // Memory for the quantization parameter of the intermediate results + memory += layer_ptr->result.dtype->tensor_params_size; + + // Memory for trainable parameter + if(layer_ptr->sizeof_paramem != 0) + { + memory += layer_ptr->sizeof_paramem(layer_ptr); + } + + layer_ptr = layer_ptr->output_layer; + } + return memory; +} + +void aialgo_distribute_parameter_memory(aimodel_t *model, void *memory_ptr, uint32_t memory_size) +{ + int i; + ailayer_t *layer_ptr = model->input_layer; + uint32_t address_counter = 0; + + for(i = 0; i < model->layer_count; i++) + { + layer_ptr->calc_result_shape(layer_ptr); + + // Memory for the quantization parameter of the intermediate results + if(layer_ptr->result.dtype->tensor_params_size != 0){ + layer_ptr->result.tensor_params = memory_ptr + address_counter; + address_counter += layer_ptr->result.dtype->tensor_params_size; + } + + // Memory for trainable parameter + if(layer_ptr->sizeof_paramem != 0) + { + layer_ptr->set_paramem(layer_ptr, memory_ptr + address_counter); + address_counter += layer_ptr->sizeof_paramem(layer_ptr); + } + + layer_ptr = layer_ptr->output_layer; + } + return; +} + + +uint8_t aialgo_schedule_inference_memory(aimodel_t *model, void *memory_ptr, uint32_t memory_size) +{ + uint16_t i; + ailayer_t *layer_ptr = model->input_layer; + + // Init result tensor with shape and memory + for(i = 0; i < model->layer_count; i++) + { + layer_ptr->calc_result_shape(layer_ptr); + layer_ptr->result.data = memory_ptr + (i % 2) * memory_size / 2; + + layer_ptr = layer_ptr->output_layer; + } + + return 0; +} + +aitensor_t *aialgo_forward_model(aimodel_t *model, aitensor_t *input_data) +{ + uint16_t i; + ailayer_t *layer_ptr = model->input_layer; + + model->input_layer->result.data = input_data->data; + model->input_layer->result.tensor_params = input_data->tensor_params; + for(i = 0; i < model->layer_count; i++) + { + layer_ptr->forward(layer_ptr); + + // Print intermediate results + //print_aitensor(&layer_ptr->result); + + layer_ptr = layer_ptr->output_layer; + } + + return &(model->output_layer->result); +} + +aitensor_t *aialgo_inference_model(aimodel_t *model, aitensor_t *input_data, aitensor_t *output_data) +{ + uint32_t i, j; + + uint16_t input_batch_shape[input_data->dim]; + aitensor_t input_batch = { + .dtype = input_data->dtype, + .shape = input_batch_shape, + .dim = input_data->dim, + .tensor_params = input_data->tensor_params + }; + aitensor_t *output_batch; + + uint32_t input_multiplier = 1; + for(i = input_data->dim - 1; i > 0; i--) + { + input_multiplier *= input_data->shape[i]; + input_batch_shape[i] = input_data->shape[i]; + } + input_batch_shape[0] = 1; + + uint32_t output_multiplier = 1; + for(i = output_data->dim - 1; i > 0; i--) + { + output_multiplier *= output_data->shape[i]; + } + + for(i = 0; i < input_data->shape[0]; i++) + { + input_batch.data = input_data->data + i * input_multiplier * input_data->dtype->size; + + output_batch = aialgo_forward_model(model, &input_batch); + + // ToDo: Copy tensor + for(j = 0; j < aimath_tensor_elements(output_batch); j++) + { + memcpy(output_data->data + i * output_multiplier * input_data->dtype->size, + output_batch->data, + aimath_tensor_elements(output_batch) * input_data->dtype->size); + } + } + memcpy(output_data->tensor_params, output_batch->tensor_params, output_batch->dtype->tensor_params_size); + return output_data; +} + +uint8_t aialgo_compile_model(aimodel_t *model) +{ + ailayer_t *layer_ptr = model->input_layer; + uint16_t layer_counter = 1; + const uint16_t MAX_LAYER_COUNT = 128; // May be an other value + + model->trainable_params_count = 0; + while(layer_ptr != model->output_layer && layer_counter < MAX_LAYER_COUNT) + { + layer_counter++; + model->trainable_params_count += layer_ptr->trainable_params_count; + + layer_ptr = layer_ptr->output_layer; + } + model->layer_count = layer_counter; + + return 0; +} + +void aialgo_print_model_structure(aimodel_t *model) +{ + int i; + ailayer_t *layer_ptr = model->input_layer; + + printf("Layers:\n"); + for(i = 0; i < model->layer_count; i++){ + if(layer_ptr->layer_type->print_specs != 0){ + printf("%4d: %s (%s) <", i + 1, layer_ptr->layer_type->name, layer_ptr->result.dtype->name); + layer_ptr->layer_type->print_specs(layer_ptr, printf); + printf(">\n"); + } else { + printf("%4d: No specification found for this layer.\n", i + 1); + } + layer_ptr = layer_ptr->output_layer; + } + return; +} + diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aialgo/aialgo_sequential_inference.h b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aialgo/aialgo_sequential_inference.h new file mode 100644 index 0000000..7b448b9 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aialgo/aialgo_sequential_inference.h @@ -0,0 +1,145 @@ +/** + * \file basic/base/aialgo/aialgo_sequential_inference.h + * \internal + * \date 20.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Functions required for inference of models + * \details The functions target memory allocation/scheduling, the calculation of the forward pass and quantization for model inference + */ + +#ifndef AIALGO_SEQUENTIAL_INFERENCE +#define AIALGO_SEQUENTIAL_INFERENCE + +#include "core/aifes_core.h" +#include "core/aifes_math.h" +#include "basic/base/aimath/aimath_basic.h" +#include "basic/default/ailayer/ailayer_dense_default.h" + +/** @brief Calculate the memory requirements for intermediate results of an inference + * + * This memory is mainly for the result buffers of the layers. + * + * Use aialgo_schedule_inference_memory() to set the memory to the model. + * + * @param *model The model + * @return Required memory size in bytes + */ +uint32_t aialgo_sizeof_inference_memory(aimodel_t *model); + +/** @brief Calculate the memory requirements for the trainable parameters (like weights, bias, ...) of the model + * + * Use aialgo_distribute_parameter_memory() to set the memory to the model. + * + * @param *model The model + * @return Required memory size in bytes + */ +uint32_t aialgo_sizeof_parameter_memory(aimodel_t *model); + +/** @brief Assign the memory for intermediate results of an inference to the model + * + * The required memory size can be calculated with aialgo_sizeof_inference_memory() + * + * @param *model The model + * @param *memory_ptr Pointer to the memory block + * @param memory_size Size of the memory block (for error checking) + * @return 0 if successful + */ +uint8_t aialgo_schedule_inference_memory(aimodel_t *model, void *memory_ptr, uint32_t memory_size); + +/** @brief Assign the memory for the trainable parameters (like weights, bias, ...) of the model + * + * Only use this function if the parameters are not pre-trained or manually configured. + * Afterwards an initialization of the memory (for example by initializing the weights) has to be performed. + * + * The required memory size can be calculated with aialgo_sizeof_parameter_memory() + * + * @param *model The model + * @param *memory_ptr Pointer to the memory block + * @param memory_size Size of the memory block (for error checking) + * @return 0 if successful + */ +void aialgo_distribute_parameter_memory(aimodel_t *model, void *memory_ptr, uint32_t memory_size); + +/** @brief Perform a forward pass on the model + * + * The result is stored in the result tensor of the output layer and a pointer to this is returned. + * This output result is stored in the inference memory and is only valid as long as the inference memory is valid. + * To get the output as a separate tensor, use aialgo_inference_model() instead. + * + * @param *model The model + * @param *input_data Input data tensor of the same shape as the input_layer shape + * @return Pointer to the output data of the forward pass (points to the result tensor of the output layer) + */ +aitensor_t *aialgo_forward_model(aimodel_t *model, aitensor_t *input_data); + +/** @brief Perform an inference on the model / Run the model + * + * Make shure to initialize the model (aialgo_compile_model()) and schedule the inference memory + * (for example with aialgo_schedule_inference_memory() or aialgo_schedule_training_memory()) before + * calling this function. + * + * Example: + * \code{.c} + * float input_data[] = {0.0f, 1.0f}; + * uint16_t input_shape[] = {1, 2} + * aitensor_t input_tensor = { + * .dtype = aif32, + * .dim = 2, + * .shape = input_shape, + * .data = input_data + * }; + * + * float output_data[1]; + * uint16_t output_shape[] = {1, 1} + * aitensor_t output_tensor = { + * .dtype = aif32, + * .dim = 2, + * .shape = output_shape, + * .data = output_data + * }; + * + * aialgo_inference_model(&model, &input_tensor, &output_tensor); + * + * // The results are now in the output_tensor + * \endcode + * + * @param *model The model + * @param *input_data Input data tensor of the same shape as the input_layer shape + * @param *output_data Empty tensor for the results of the inference with the size of your outputs + * @return Pointer to the output_data tensor with the results + */ +aitensor_t *aialgo_inference_model(aimodel_t *model, aitensor_t *input_data, aitensor_t *output_data); + +/** @brief Initialize the model structure +* +* Counts the number of layers and trainable parameters in a model as preparation for inference or training. +* +* @param *model The model +* @return 0 if successful +*/ +uint8_t aialgo_compile_model(aimodel_t *model); + +/** @brief Print the layer structure of the model with the configured parameters +* +* @param *model The model +*/ +void aialgo_print_model_structure(aimodel_t *model); + +#endif // AIALGO_SEQUENTIAL_INFERENCE diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aialgo/aialgo_sequential_training.c b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aialgo/aialgo_sequential_training.c new file mode 100644 index 0000000..cae7d3f --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aialgo/aialgo_sequential_training.c @@ -0,0 +1,299 @@ +/** + * \file basic/base/aialgo/aialgo_sequential_training.c + * \version 2.0alpha + * \date 20.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/aialgo/aialgo_sequential_training.h" +#include "basic/base/aialgo/aialgo_sequential_inference.h" + +// ToDo: Remove dependency +#include "basic/default/aimath/aimath_f32_default.h" + +uint32_t aialgo_sizeof_training_memory(aimodel_t *model, aiopti_t *optimizer) +{ + uint16_t i, j; + ailayer_t *layer_ptr = model->input_layer; + uint32_t memory = 0; + + for(i = 0; i < model->layer_count; i++) + { + // Result memory + layer_ptr->calc_result_shape(layer_ptr); + memory += aimath_sizeof_tensor_data(&(layer_ptr->result)); + + // Memory for the qantization parameter of the deltas + if(layer_ptr->output_layer->deltas.dtype != 0){ + memory += layer_ptr->output_layer->deltas.dtype->tensor_params_size; + } + + // Trainingmemory e.g. for gradients + if(layer_ptr->sizeof_trainmem != 0) + { + memory += layer_ptr->sizeof_trainmem(layer_ptr); + } + + // optimization memory (e.g. first or second momentum) + if(optimizer->sizeof_optimem != 0){ + for(j = 0; j < layer_ptr->trainable_params_count; j++){ + memory += optimizer->sizeof_optimem(optimizer, layer_ptr->trainable_params[j]); + } + } + + layer_ptr = layer_ptr->output_layer; + } + return memory; +} + +uint8_t aialgo_schedule_training_memory(aimodel_t *model, aiopti_t *optimizer, void *memory_ptr, uint32_t memory_size) +{ + uint16_t i, j; + uint32_t address_counter = 0; + ailayer_t *layer_ptr = model->input_layer; + + for(i = 0; i < model->layer_count; i++) + { + // Result memory = deltas memory + layer_ptr->calc_result_shape(layer_ptr); + layer_ptr->result.data = memory_ptr + address_counter; + + layer_ptr->output_layer->deltas.dtype = layer_ptr->result.dtype; + layer_ptr->output_layer->deltas.dim = layer_ptr->result.dim; + layer_ptr->output_layer->deltas.shape = layer_ptr->result.shape; + layer_ptr->output_layer->deltas.data = memory_ptr + address_counter; + address_counter += aimath_sizeof_tensor_data(&(layer_ptr->result)); + + // Memory for the qantization parameter of the deltas + if(layer_ptr->output_layer->deltas.dtype != 0){ + layer_ptr->output_layer->deltas.tensor_params = memory_ptr + address_counter; + address_counter += layer_ptr->output_layer->deltas.dtype->tensor_params_size; + } + + // Training memory e.g. for gradients + if(layer_ptr->sizeof_trainmem != 0) + { + layer_ptr->set_trainmem(layer_ptr, memory_ptr + address_counter); + address_counter += layer_ptr->sizeof_trainmem(layer_ptr); + } + + // optimization memory (e.g. first or second momentum) + if(optimizer->sizeof_optimem != 0){ + for(j = 0; j < layer_ptr->trainable_params_count; j++){ + layer_ptr->optimem[j] = memory_ptr + address_counter; + address_counter += optimizer->sizeof_optimem(optimizer, layer_ptr->trainable_params[j]); + } + } + + layer_ptr = layer_ptr->output_layer; + } + + return 0; +} + +void aialgo_init_model_for_training(aimodel_t *model, aiopti_t *optimizer) +{ + uint16_t i, j; + ailayer_t *layer_ptr = model->input_layer; + + for(i = 0; i < model->layer_count; i++) + { + // Init the optimization memory (e.g. setting the momentums to zero) + for(j = 0; j < layer_ptr->trainable_params_count; j++){ + if(optimizer->init_optimem != 0){ + optimizer->init_optimem(optimizer, layer_ptr->trainable_params[j], layer_ptr->gradients[j], layer_ptr->optimem[j]); + } + } + layer_ptr = layer_ptr->output_layer; + } + return; +} + +void aialgo_backward_model(aimodel_t *model, aitensor_t *target_data) +{ + uint16_t i; + ailayer_t *layer_ptr = model->output_layer; + + model->loss->calc_delta(model->loss, target_data); + for(i = 0; i < model->layer_count; i++) + { +#ifdef DEBUG_CHECKS + if(layer_ptr->backward == 0){ + printf("\nError: No backward function implementation in layer %d\n", i); + return; + } +#endif + layer_ptr->backward(layer_ptr); + layer_ptr = layer_ptr->input_layer; + } + return; +} + +void aialgo_train_model(aimodel_t *model, aitensor_t *input_tensor, aitensor_t *target_tensor, aiopti_t *optimizer, uint32_t batch_size) +{ + uint32_t i; + + aitensor_t input_batch; + uint16_t input_batch_shape[input_tensor->dim]; + input_batch.dtype = input_tensor->dtype; + input_batch.dim = input_tensor->dim; + input_batch.shape = input_batch_shape; + input_batch.tensor_params = input_tensor->tensor_params; + aitensor_t target_batch; + uint16_t target_batch_shape[target_tensor->dim]; + target_batch.dtype = target_tensor->dtype; + target_batch.dim = target_tensor->dim; + target_batch.shape = target_batch_shape; + target_batch.tensor_params = target_tensor->tensor_params; + + uint32_t input_multiplier = 1; + for(i = input_tensor->dim - 1; i > 0; i--) + { + input_multiplier *= input_tensor->shape[i]; + input_batch_shape[i] = input_tensor->shape[i]; + } + input_multiplier *= input_tensor->dtype->size; + input_batch_shape[0] = 1; + uint32_t target_multiplier = 1; + for(i = target_tensor->dim - 1; i > 0; i--) + { + target_multiplier *= target_tensor->shape[i]; + target_batch_shape[i] = target_tensor->shape[i]; + } + target_multiplier *= target_tensor->dtype->size; + target_batch_shape[0] = 1; + + uint32_t batch_count = (uint32_t) (input_tensor->shape[0] / batch_size); + uint32_t batch; + for(batch = 0; batch < batch_count; batch++) + { + aialgo_zero_gradients_model(model, optimizer); + for(i = 0; i < batch_size; i++) + { + input_batch.data = input_tensor->data + batch * input_multiplier * batch_size + i * input_multiplier; + target_batch.data = target_tensor->data + batch * target_multiplier * batch_size + i * target_multiplier; + + aialgo_forward_model(model, &input_batch); + aialgo_backward_model(model, &target_batch); + } + aialgo_update_params_model(model, optimizer); + } + return; +} + +void aialgo_calc_loss_model_f32(aimodel_t *model, aitensor_t *input_tensor, aitensor_t *target_tensor, float *result) +{ + uint32_t i; + float loss; + + aitensor_t input_batch; + uint16_t input_batch_shape[input_tensor->dim]; + input_batch.dtype = input_tensor->dtype; + input_batch.dim = input_tensor->dim; + input_batch.shape = input_batch_shape; + input_batch.tensor_params = input_tensor->tensor_params; + aitensor_t target_batch; + uint16_t target_batch_shape[target_tensor->dim]; + target_batch.dtype = target_tensor->dtype; + target_batch.dim = target_tensor->dim; + target_batch.shape = target_batch_shape; + target_batch.tensor_params = target_tensor->tensor_params; + + uint32_t input_multiplier = 1; + for(i = input_tensor->dim - 1; i > 0; i--) + { + input_multiplier *= input_tensor->shape[i]; + input_batch_shape[i] = input_tensor->shape[i]; + } + input_multiplier *= input_tensor->dtype->size; + input_batch_shape[0] = 1; + uint32_t target_multiplier = 1; + for(i = target_tensor->dim - 1; i > 0; i--) + { + target_multiplier *= target_tensor->shape[i]; + target_batch_shape[i] = target_tensor->shape[i]; + } + target_multiplier *= target_tensor->dtype->size; + target_batch_shape[0] = 1; + + *result = 0; + for(i = 0; i < input_tensor->shape[0]; i++) + { + input_batch.data = input_tensor->data + i * input_multiplier; + target_batch.data = target_tensor->data + i * target_multiplier; + + aialgo_forward_model(model, &input_batch); + model->loss->calc_loss(model->loss, &target_batch, &loss); + *result += loss; + } + return; +} + +void aialgo_zero_gradients_model(aimodel_t *model, aiopti_t *optimizer) +{ + uint16_t i, j; + ailayer_t *layer_ptr = model->input_layer; + + for(i = 0; i < model->layer_count; i++) + { + for(j = 0; j < layer_ptr->trainable_params_count; j++){ + optimizer->zero_gradients(optimizer, layer_ptr->gradients[j]); + } + layer_ptr = layer_ptr->output_layer; + } + return; +} + +void aialgo_update_params_model(aimodel_t *model, aiopti_t *optimizer) +{ + uint16_t i, j; + ailayer_t *layer_ptr = model->input_layer; + + if(optimizer->begin_step != 0){ + optimizer->begin_step(optimizer); + } + for(i = 0; i < model->layer_count; i++) + { + for(j = 0; j < layer_ptr->trainable_params_count; j++){ + optimizer->update_params(optimizer, layer_ptr->trainable_params[j], layer_ptr->gradients[j], layer_ptr->optimem[j]); + } + layer_ptr = layer_ptr->output_layer; + } + if(optimizer->end_step != 0){ + optimizer->end_step(optimizer); + } + return; +} + +void aialgo_print_loss_specs(ailoss_t *loss) +{ + printf("%s (%s) <", loss->loss_type->name, loss->connection_layer.deltas.dtype->name); + loss->loss_type->print_specs(loss, printf); + printf(">"); + return; +} + +void aialgo_print_optimizer_specs(aiopti_t *opti) +{ + printf("%s (%s) <", opti->optimizer_type->name, opti->dtype->name); + opti->optimizer_type->print_specs(opti, printf); + printf(">"); + return; +} diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aialgo/aialgo_sequential_training.h b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aialgo/aialgo_sequential_training.h new file mode 100644 index 0000000..5ba2aa4 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aialgo/aialgo_sequential_training.h @@ -0,0 +1,148 @@ +/** + * \file basic/base/aialgo/aialgo_sequential_training.h + * \internal + * \date 20.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Functions required for the training of models + * \details The functions target memory allocation/scheduling and the backpropagation for model training + */ + +#ifndef AIALGO_SEQUENTIAL_TRAINING +#define AIALGO_SEQUENTIAL_TRAINING + +#include "core/aifes_core.h" +#include "core/aifes_math.h" +#include "basic/base/aimath/aimath_basic.h" + +/** @brief Calculate the memory requirements for model training + * + * This memory is used for intermediate results, gradients and momentums. + * + * Use aialgo_schedule_training_memory() to set the memory to the model. + * + * @param *model The model + * @param *optimizer The optimizer that is used for training + * @return Required memory size in bytes + */ +uint32_t aialgo_sizeof_training_memory(aimodel_t *model, aiopti_t *optimizer); + +/** @brief Assign the memory for model training + * + * This memory is used for intermediate results, gradients and momentums. + * + * The required memory size can be calculated with aialgo_sizeof_training_memory(). + * + * @param *model The model + * @param *optimizer The optimizer that is used for training + * @param *memory_ptr Pointer to the memory block + * @param memory_size Size of the memory block (for error checking) + * @return 0 if successful + */ +uint8_t aialgo_schedule_training_memory(aimodel_t *model, aiopti_t *optimizer, void *memory_ptr, uint32_t memory_size); + +/** @brief Initialize the optimization memory of the model layers + * + * @param *model The model + * @param *optimizer The optimizer that is used for training + */ +void aialgo_init_model_for_training(aimodel_t *model, aiopti_t *optimizer); + +/** @brief Perform the backward pass + * + * @param *model The model + * @param *target_data The tensor containing the target data / labels + */ +void aialgo_backward_model(aimodel_t *model, aitensor_t *target_data); + +/** @brief Perform one training epoch on all data batches of the dataset using backpropagation + * + * Make shure to initialize the model (aialgo_compile_model()) and schedule the training memory + * (for example with aialgo_schedule_training_memory()) and initialize the training memory + * (aialgo_init_model_for_training()) before calling this function. + * + * Example: Training of an F32 model for multiple epochs + * \code{.c} + * int epochs = 100; + * int batch_size = 4; + * int print_interval = 10; + * + * float loss; + * for(i = 0; i < epochs; i++) + * { + * // One epoch of training. Iterates through the whole data once + * aialgo_train_model(&model, &input_tensor, &target_tensor, optimizer, batch_size); + + * // Calculate and print loss every print_interval epochs + * if(i % print_interval == 0) + * { + * aialgo_calc_loss_model_f32(&model, &input_tensor, &target_tensor, &loss); + * printf("Epoch %5d: loss: %f\n", i, loss); + * } + * } + * \endcode + * + * @param *model The model + * @param *input_tensor The tensor containing the input data + * @param *target_tensor The tensor containing the target data / labels + * @param *optimizer The optimizer that is used for training + * @param batch_size Size of a batch / Number of input vektors + */ +void aialgo_train_model(aimodel_t *model, aitensor_t *input_tensor, aitensor_t *target_tensor, aiopti_t *optimizer, uint32_t batch_size); + +/** @brief Calculate the loss in \link aimath_f32.h F32 \endlink data type + * + * @param *model The model + * @param *input_data Tensor containing the input data + * @param *target_data Tensor containing the target data / labels + * @param *result The calculated loss will be written here + */ +void aialgo_calc_loss_model_f32(aimodel_t *model, aitensor_t *input_data, aitensor_t *target_data, float *result); + +/** @brief Set the gradients to zero + * + * @param *model The model + * @param *optimizer The optimizer that is used for training + */ +void aialgo_zero_gradients_model(aimodel_t *model, aiopti_t *optimizer); + +/** @brief Perform the optimization step on the model parameters + * + * @param *model The model + * @param *optimizer The optimizer that is used for training + */ +void aialgo_update_params_model(aimodel_t *model, aiopti_t *optimizer); + +/** @brief Print the loss specs + * + * Prints information like type, data type and constants to the console. + * + * @param *loss The loss + */ +void aialgo_print_loss_specs(ailoss_t *loss); + +/** @brief Print the optimizer specs + * + * Prints information like type, data type and constants to the console. + * + * @param *opti The optimizer + */ +void aialgo_print_optimizer_specs(aiopti_t *opti); + +#endif // AIALGO_SEQUENTIAL_TRAINING diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_dense.c b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_dense.c new file mode 100644 index 0000000..d3306b4 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_dense.c @@ -0,0 +1,246 @@ +/** + * \file basic/base/ailayer/ailayer_dense.c + * \version 2.0alpha + * \date 20.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/ailayer/ailayer_dense.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_layertype_t ailayer_dense_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "Dense", + .print_specs = ailayer_dense_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_layertype_t *ailayer_dense_type = &ailayer_dense_type_s; + +ailayer_t *ailayer_dense(ailayer_dense_t *layer, ailayer_t *input_layer) +{ + layer->base.layer_type = ailayer_dense_type; + + layer->base.input_layer = input_layer; + input_layer->output_layer = &(layer->base); + + layer->base.layer_configuration = layer; + layer->base.result.dtype = layer->result_dtype; + layer->base.result.dim = 2; + layer->base.result.shape = layer->result_shape; + layer->base.result.shape[1] = layer->neurons; + + layer->base.deltas.dtype = layer->result_dtype; + layer->base.deltas.dim = 2; + layer->base.deltas.shape = input_layer->result.shape; + + layer->weights.dim = 2; + layer->weights.dtype = layer->weights_dtype; + layer->weights.shape = layer->weights_shape; + layer->weights.shape[0] = input_layer->result.shape[1]; + layer->weights.shape[1] = layer->neurons; + + layer->bias.dim = 2; + layer->bias.dtype = layer->bias_dtype; + layer->bias.shape = layer->bias_shape; + layer->bias.shape[0] = 1; + layer->bias.shape[1] = layer->neurons; + + layer->base.forward = ailayer_dense_forward; + layer->base.backward = ailayer_dense_backward; + + layer->base.calc_result_shape = ailayer_dense_calc_result_shape; + layer->base.sizeof_paramem = ailayer_dense_sizeof_paramem; + layer->base.set_paramem = ailayer_dense_set_paramem; + layer->base.sizeof_trainmem = ailayer_dense_sizeof_trainmem; + layer->base.set_trainmem = ailayer_dense_set_trainmem; + + layer->base.get_result_bound = 0; + + layer->base.trainable_params_count = 2; + layer->base.trainable_params = layer->trainable_params; + layer->base.gradients = layer->gradients; + layer->base.optimem = layer->optimem; + + layer->trainable_params[0] = &layer->weights; + layer->trainable_params[1] = &layer->bias; + + return &layer->base; +} + +void ailayer_dense_forward(ailayer_t *self) +{ + aitensor_t *input_tensor = &(self->input_layer->result); + aitensor_t *result_tensor = &(self->result); + ailayer_dense_t *layer = (ailayer_dense_t *)(self->layer_configuration); + aitensor_t *weight_tensor = &(layer->weights); + aitensor_t *bias_tensor = &(layer->bias); + + // z = x * W + b + layer->linear(input_tensor, weight_tensor, bias_tensor, result_tensor); + + return; +} + + +void ailayer_dense_backward(ailayer_t *self) +{ + aitensor_t *delta_in = &(self->deltas); + aitensor_t *delta_out = &(self->output_layer->deltas); + aitensor_t *x_in = &(self->input_layer->result); + ailayer_dense_t *layer = (ailayer_dense_t *)(self->layer_configuration); + aitensor_t *weights = &(layer->weights); + aitensor_t *d_weights = layer->gradients[0]; + aitensor_t *d_bias = layer->gradients[1]; + + int8_t temp_result_data[aimath_sizeof_tensor_data(d_weights)]; + aitensor_t temp_result = { + .dim = 2, + .shape = d_weights->shape, + .data = temp_result_data, + .dtype = d_weights->dtype, + .tensor_params = d_weights->tensor_params + }; + + aimath_transpose_vector(x_in); + // d_weights += x_in^T * delta_out + layer->mat_mul(x_in, delta_out, &temp_result); + layer->tensor_add(d_weights, &temp_result, d_weights); + aimath_transpose_vector(x_in); + // d_bias += delta_out + layer->tensor_add(d_bias, delta_out, d_bias); + + // Calculate delta for next layer. Do not before calculating gradients!!! May override x_in!!! + // d_in = w^T * d_out = (d_out^T * w)^T + aimath_transpose_vector(delta_out); + aimath_transpose_vector(delta_in); + layer->mat_mul(weights, delta_out, delta_in); + aimath_transpose_vector(delta_in); + aimath_transpose_vector(delta_out); + + return; +} + +void ailayer_dense_calc_result_shape(ailayer_t *self) +{ + ailayer_dense_t *layer = (ailayer_dense_t *)(self->layer_configuration); + + self->result.shape[0] = self->input_layer->result.shape[0]; + self->result.shape[1] = layer->neurons; + + return; +} + +uint32_t ailayer_dense_sizeof_paramem(const ailayer_t *self) +{ + uint32_t memory = 0; + ailayer_dense_t *layer = (ailayer_dense_t *)(self->layer_configuration); + + // Weights + memory += layer->weights_dtype->tensor_params_size; + memory += self->input_layer->result.shape[1] * layer->neurons * aimath_sizeof_dtype(layer->weights_dtype); // data + + // Bias + memory += layer->bias_dtype->tensor_params_size; + memory += layer->neurons * aimath_sizeof_dtype(layer->bias_dtype); // data + return memory; +} + +void ailayer_dense_set_paramem(ailayer_t *self, void *memory_ptr) +{ + uint32_t address_counter = 0; + ailayer_dense_t *layer = (ailayer_dense_t *) (self->layer_configuration); + + layer->weights.tensor_params = memory_ptr + address_counter; + address_counter += layer->weights_dtype->tensor_params_size; + layer->weights.dim = 2; + layer->weights.dtype = layer->weights_dtype; + layer->weights.shape = layer->weights_shape; + layer->weights.shape[0] = self->input_layer->result.shape[1]; + layer->weights.shape[1] = layer->neurons; + layer->weights.data = memory_ptr + address_counter; + address_counter += aimath_sizeof_tensor_data(&(layer->weights)); + + layer->bias.tensor_params = memory_ptr + address_counter; + address_counter += layer->bias_dtype->tensor_params_size; + layer->bias.dim = 2; + layer->bias.dtype = layer->bias_dtype; + layer->bias.shape = layer->bias_shape; + layer->bias.shape[0] = 1; + layer->bias.shape[1] = layer->neurons; + layer->bias.data = memory_ptr + address_counter; + //address_counter += aimath_sizeof_tensor_data(&(configuration->bias)); + + layer->trainable_params[0] = &(layer->weights); + layer->trainable_params[1] = &(layer->bias); + + return; +} + +uint32_t ailayer_dense_sizeof_trainmem(const ailayer_t *self) +{ + uint32_t memory = 0; + ailayer_dense_t *layer = (ailayer_dense_t *)(self->layer_configuration); + + memory += aimath_sizeof_tensor(&layer->weights); + memory += aimath_sizeof_tensor(&layer->bias); + return memory; +} + +void ailayer_dense_set_trainmem(ailayer_t *self, void *memory_ptr) +{ + uint32_t address_counter = 0; + ailayer_dense_t *layer = (ailayer_dense_t *) (self->layer_configuration); + + // Weights gradients in gradients[0] + self->gradients[0] = memory_ptr; + address_counter += sizeof(aitensor_t); + self->gradients[0]->data = memory_ptr + address_counter; + self->gradients[0]->dtype = layer->weights.dtype; + self->gradients[0]->dim = 2; + self->gradients[0]->shape = layer->weights.shape; + address_counter += aimath_sizeof_tensor_data(layer->gradients[0]); + self->gradients[0]->tensor_params = memory_ptr + address_counter; + address_counter += aimath_sizeof_tensor_params(layer->gradients[0]); + + // Bias gradients in gradients[1] + self->gradients[1] = memory_ptr + address_counter; + address_counter += sizeof(aitensor_t); + self->gradients[1]->data = memory_ptr + address_counter; + self->gradients[1]->dtype = layer->bias.dtype; + self->gradients[1]->dim = 2; + self->gradients[1]->shape = layer->bias.shape; + address_counter += aimath_sizeof_tensor_data(layer->gradients[1]); + self->gradients[1]->tensor_params = memory_ptr + address_counter; + address_counter += aimath_sizeof_tensor_params(layer->gradients[1]); + + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void ailayer_dense_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)) +{ + ailayer_dense_t *layer = (ailayer_dense_t *)(self->layer_configuration); + + print("neurons: %ld", (long unsigned int) layer->neurons); +} +#endif diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_dense.h b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_dense.h new file mode 100644 index 0000000..66cbd2d --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_dense.h @@ -0,0 +1,296 @@ +/** + * \file basic/base/ailayer/ailayer_dense.h + * \internal + * \date 20.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link ailayer layer \endlink implementation of the Dense layer + * + * This is an "abstract" data-type independent implementation. To use the layer use one of the provided + * implementations for a specific hardware and data-type (for example from ailayer_dense_default.h) or set + * the required math functions on your own. + * + * \image html ailayer_dense_schematic.png width=200px + * + * The Dense layer (or fully connected layer) is the core layer of a FNN and calculates the weighted sums of its inputs + * @f[ + * y = x \cdot W + b + * @f] + * with the number of inputs \f$ K \f$, the number of neurons / outputs \f$ M \f$, + * the input vektor \f$ x \in \mathbb{R}^{1 \times K} \f$, the output vektor \f$ y \in \mathbb{R}^{1 \times M} \f$, + * the weights matrix \f$ W \in \mathbb{R}^{K \times M} \f$ and the bias \f$ b \in \mathbb{R}^{1 \times M} \f$. + * + * To increase the computational speed it is also possible to process a whole data batch at ones: + * @f[ + * Y = X \cdot W \oplus b + * @f] + * with the number of inputs \f$ K \f$, the number of neurons / outputs \f$ M \f$, the batch size \f$ N \f$, + * the input matrix \f$ X \in \mathbb{R}^{N \times K} \f$, the output vektor \f$ Y \in \mathbb{R}^{N \times M} \f$, + * the weights matrix \f$ W \in \mathbb{R}^{K \times M} \f$ and the bias \f$ b \in \mathbb{R}^{1 \times M} \f$. + * In this case the \f$ \oplus \f$ means a broadcasted addition of the bias to the rows of the result of \f$ X \cdot W \f$: + * @f[ + * X \cdot W \oplus b \rightarrow X \cdot W + \left( \begin{array}{c} 1 \\ \vdots \\ 1 \\ \end{array}\right) \cdot b + * @f] + * Keep in mind that also the memory size for intermediate results increases with the batch size. + * + * The results of the forward pass of this layer are written to the result tensor of the base ailayer_t struct. + */ + +#ifndef AILAYER_DENSE +#define AILAYER_DENSE + +#include "core/aifes_core.h" +#include "basic/default/aimath/aimath_f32_default.h" + +#define DENSE_WEIGHTS_SIZE(INPUTS, OUTPUTS) ((INPUTS) * (OUTPUTS)) +#define DENSE_BIAS_SIZE(OUTPUTS) (OUTPUTS) + +#define DENSE_WEIGHTS_SHAPE(INPUTS, OUTPUTS) {INPUTS, OUTPUTS} +#define DENSE_BIAS_SHAPE(OUTPUTS) {1, OUTPUTS} + +typedef struct ailayer_dense ailayer_dense_t; + +/** @brief General \link ailayer_dense.h Dense layer \endlink structure +* +*/ +struct ailayer_dense { + ailayer_t base; /**< Inherited field members from general ailayer struct. */ + const aimath_dtype_t *result_dtype; /**< Data type of the inference result values. */ + + /** @name Layer configuration + * @brief Required configuration parameters for the layer + * + * These fields have to be configured by the user before calling the initializer function. + */ + ///@{ + uint32_t neurons; /**< Layer neurons count (number of outputs). */ + ///@} + + /** @name Trainable parameters + * @brief Data fields for the trainable parameters (weights, bias) of the layer + */ + ///@{ + aitensor_t weights; /**< Tensor containing the layer weights. */ + aitensor_t bias; /**< Tensor containing the layer bias weights. */ + + const aimath_dtype_t *weights_dtype; /**< Data type of the weights. */ + const aimath_dtype_t *bias_dtype; /**< Data type of the bias weights. */ + + uint16_t weights_shape[2]; /**< Weights tensor shape (n x m matrix). */ + uint16_t bias_shape[2]; /**< Bias weights tensor shape (n x m matrix). */ + + aitensor_t *trainable_params[2]; /**< Pointer to the weights and biases (which are the trainable parameters). */ + aitensor_t *gradients[2]; /**< Gradients structure for the back propagation algorithm. */ + void *optimem[2]; /**< Memory field used by the trainings optimizer. */ + ///@} + + /** @name Math functions + * @brief Required data type specific math functions + */ + ///@{ + + /** @brief Required math function: Linear transformation + * + * Requires a math function that performs a linear transformation:\n + * @f[ + * result = a \cdot b \oplus c = a \cdot b + \left( \begin{array}{c} 1 \\ \vdots \\ 1 \\ \end{array}\right) \cdot c + * @f] + * + * @param a Matrix with dimension \f$ N \times K \f$ (input) + * @param b Matrix with dimension \f$ K \times M \f$ (input) + * @param c Laying vektor with dimension \f$ 1 \times M \f$ (input) + * @param result Matrix with dimension \f$ N \times M \f$ (output) + */ + void (*linear)(const aitensor_t *a, const aitensor_t *b, const aitensor_t *c, aitensor_t *result); + + /** @brief Required math function: Matrix multiplication + * + * Requires a math function that performs a matrix multiplication on two 2D tensors:\n + * @f[ + * result = a \cdot b + * @f] + * + * @param a Matrix with dimension \f$ N \times K \f$ (input) + * @param b Matrix with dimension \f$ K \times M \f$ (input) + * @param result Matrix with dimension \f$ N \times M \f$ (output) + */ + void (*mat_mul)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + /** @brief Required math function: Element wise tensor addition + * + * Requires a math function that adds two tensors element wise:\n + * @f[ + * result = a + b + * @f] + */ + void (*tensor_add)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + ///@} + + uint16_t result_shape[2]; /**< Inference result tensor (ailayer.result) shape. */ +}; + +/** @brief Dense layer type + * + * Defines the type of the layer (for example for type checks and debug prints). + * See aicore_layertype for more information about the layer type. + */ +extern const aicore_layertype_t *ailayer_dense_type; + +/** @brief Initialize and connect the given Dense layer + * + * This function represents the "constructor" of the abstract Dense layer. It initializes the layer structure + * and connects it to the previous layer.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example ailayer_dense_f32_default()). + * + * @param *layer The layer to initialize. + * @param *input_layer The previous layer that provides the inputs to the layer. + * @return Pointer to the (successfully) initialized general layer structure (ailayer_dense.base) + */ +ailayer_t *ailayer_dense(ailayer_dense_t *layer, ailayer_t *input_layer); + +/** @brief Calculate the forward pass for given Dense layer + * + * *Implementation of ailayer.forward.* + * + * It uses the result tensor of the previous layer as input and writes the result of the forward pass + * to the result tensor (ailayer.result) of the given layer. + * + * Calculation of the forward pass result: + * @f[ + * x_{out} \leftarrow x_{in} \cdot w \oplus b + * @f] + * + * \f$ w \f$: Weights matrix\n + * \f$ b \f$: Bias vektor\n + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ x_{out} \f$: Result of the forward pass of this layer\n\n + * + * Used math functions: + * * ailayer_dense.linear + * + * @param *self Layer to calculate the forward path for. + */ +void ailayer_dense_forward(ailayer_t *self); + +/** @brief Calculate the backward pass for the given Dense layer + * + * *Implementation of ailayer.backward.* + * + * It uses the deltas tensor of the next layer as input and writes the result of the backward pass + * to the deltas tensor (ailayer.deltas) of the given layer. + * + * Calculation of the gradients: + * @f[ + * \partial w \leftarrow \partial w + x_{in}^T \cdot \delta_{out} + * @f] + * @f[ + * \partial b \leftarrow \partial b + \left( \begin{array}{c} 1 & \cdots & 1 \\ \end{array}\right) \cdot \delta_{out} + * @f] + * + * Calculation of the errors for the previous layer: + * @f[ + * \delta_{in} \leftarrow w^T \cdot \delta_{out} = (\delta_{out}^T \cdot w)^T + * @f] + * + * \f$ w \f$: Weights matrix\n + * \f$ b \f$: Bias vektor\n + * \f$ \partial w \f$: Gradients matrix for the weights\n + * \f$ \partial b \f$: Gradients vektor for the bias\n + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ \delta_{in} \f$: Result of the backward pass of this layer\n + * \f$ \delta_{out} \f$: Result of the backward pass of the next layer\n\n + * + * Used math functions: + * * ailayer_dense.mat_mul + * * ailayer_dense.tensor_add + * + * @param *self Layer to calculate the backward path for. + */ +void ailayer_dense_backward(ailayer_t *self); + +/** @brief Calculate the shape of the result tensor (ailayer.result) + * + * *Implementation of ailayer.calc_result_shape.* + * + * Resulting shape is [count_inputs x count_neurons] + * + * @param *self Layer to calculate the resulting shape for. + */ +void ailayer_dense_calc_result_shape(ailayer_t *self); + + +/** @brief Calculate and return the parameter memory size needed for this layer + * + * *Implementation of ailayer.sizeof_paramem.* + * + * The parameter size is calculated for the \link ailayer_dense.weights weights \endlink and + * \link ailayer_dense.bias bias \endlink tensors, including the data and tensor_params fields. + * + * @param *self The layer to calculate the parameter memory size for + * @return Calculated parameter memory size in bytes. + */ +uint32_t ailayer_dense_sizeof_paramem(const ailayer_t *self); + +/** @brief Distribute provided memory to the parameter pointers + * + * *Implementation of ailayer.set_paramem.* + * + * Distributes the given buffer to the parameter pointers and sets + * the tensor parameters for weights and bias holding structures.\n + * The required parameter size can be calculated with ailayer_dense_sizeof_paramem() + * + * @param *self The layer to set the memory fields for. + * @param *memory_ptr The memory that can be used for the parameters + */ +void ailayer_dense_set_paramem(ailayer_t *self, void *memory_ptr); + +/** @brief Calculate and return the memory size needed by this layer for training + * + * *Implementation of ailayer.sizeof_trainmem.* + * + * The memory size is calculated for the gradient tensors of weights and bias. + * + * @param *self The layer to calculate the gradient memory size for. + * @return Calculated gradient memory size in bytes. + */ +uint32_t ailayer_dense_sizeof_trainmem(const ailayer_t *self); + +/** @brief Distribute provided memory to the gradients pointers + * + * *Implementation of ailayer.set_trainmem.* + * + * The required memory size can be calculated with ailayer_dense_sizeof_trainmem(). + * + * @param *self The layer to set the memory fields for. + * @param *memory_ptr The memory that can be used for the gradients + */ +void ailayer_dense_set_trainmem(ailayer_t *self, void *memory_ptr); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the layer specification + * + * @param *self The layer to print the specification for + * @param *print Pointer to the print function to use + */ +void ailayer_dense_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // AILAYER_DENSE diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_elu.c b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_elu.c new file mode 100644 index 0000000..35aaa1f --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_elu.c @@ -0,0 +1,113 @@ +/** + * \file basic/base/ailayer/ailayer_elu.c + * \version 2.0alpha + * \date 16.03.2021 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/ailayer/ailayer_elu.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_layertype_t ailayer_elu_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "ELU", + .print_specs = ailayer_elu_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_layertype_t *ailayer_elu_type = &ailayer_elu_type_s; + +ailayer_t *ailayer_elu(ailayer_elu_t *layer, ailayer_t *input_layer) ////const void *beta1, +{ + layer->base.layer_type = ailayer_elu_type; + + layer->base.input_layer = input_layer; + input_layer->output_layer = &(layer->base); + + layer->base.layer_configuration = layer; + layer->base.result.dtype = layer->dtype; + layer->base.result.shape = input_layer->result.shape; + layer->base.result.dim = input_layer->result.dim; + + layer->base.deltas.dtype = layer->dtype; + layer->base.deltas.dim = 2; + layer->base.deltas.shape = layer->base.result.shape; + + layer->base.forward = ailayer_elu_forward; + layer->base.backward = ailayer_elu_backward; + + layer->base.calc_result_shape = ailayer_elu_calc_result_shape; + layer->base.sizeof_paramem = 0; + layer->base.set_paramem = 0; + layer->base.sizeof_trainmem = 0; + layer->base.set_trainmem = 0; + + layer->base.trainable_params_count = 0; + + return &(layer->base); +} + +void ailayer_elu_forward(ailayer_t *self) +{ + ailayer_elu_t *layer = (ailayer_elu_t *)(self->layer_configuration); + aitensor_t *x_in = &(self->input_layer->result); + aitensor_t *x_out = &(self->result); + + layer->elu(x_in, layer->alpha, x_out); + return; +} + + +void ailayer_elu_backward(ailayer_t *self) +{ + ailayer_elu_t *layer = (ailayer_elu_t *)(self->layer_configuration); + aitensor_t *delta_in = &(self->deltas); + aitensor_t *delta_out = &(self->output_layer->deltas); + aitensor_t *x_in = &(self->input_layer->result); + + // delta_in = delta_out .* elu'(x_in) + layer->d_elu(x_in, layer->alpha, delta_in); + layer->multiply(delta_in, delta_out, delta_in); + return; +} + +void ailayer_elu_calc_result_shape(ailayer_t *self) +{ + /* Unused: Shape is already defined (Pointer) + + self->result.shape[0] = self->input_layer->result.shape[0]; + self->result.shape[1] = self->input_layer->result.shape[1]; + */ + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void ailayer_elu_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)) +{ + ailayer_elu_t *self_casted = (ailayer_elu_t *) self->layer_configuration; + + print("alpha: "); + self_casted->dtype->print_aiscalar(self_casted->alpha, print); + return; +} +#endif + diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_elu.h b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_elu.h new file mode 100644 index 0000000..4a5bde3 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_elu.h @@ -0,0 +1,200 @@ +/** + * \file basic/base/ailayer/ailayer_elu.h + * \internal + * \date 15.03.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link ailayer layer \endlink implementation of the ELU activation layer + * + * This is an "abstract" data-type independent implementation. To use the layer use one of the provided + * implementations for a specific hardware and data-type (for example from ailayer_elu_default.h) or set + * the required math functions on your own. + * + * \image html ailayer_elu_schematic.png width=200px + * + * The ELU layer is used as an activation function layer right after a dense layer. It calculates + * @f[ + * y = \begin{cases} + \alpha (e^x - 1) & \text{if } x \leq 0\\ + x & \text{if } x > 0 + \end{cases} + * @f] + * for every element of the input tensor. + * + * The results of the forward pass of this layer are written to the result tensor of the base ailayer_t struct. + */ + +#ifndef ELU_LAYER +#define ELU_LAYER + +#include "core/aifes_core.h" + +typedef struct ailayer_elu ailayer_elu_t; + +/** @brief General \link ailayer_elu.h ELU layer \endlink struct +* +*/ +struct ailayer_elu { + ailayer_t base; /**< Inherited field members from general ailayer struct. */ + const aimath_dtype_t *dtype; /**< Data type of the input and inference result values. */ + + /** @name Layer configuration + * @brief Required configuration parameters for the layer + * + * These fields have to be configured by the user before calling the initializer function. + */ + ///@{ + void *alpha; /**< Parameter \f$ \alpha \f$ used to calculate ELU function for input values < 0. */ + ///@} + + /** @name Math functions + * @brief Required data type specific math functions + */ + ///@{ + + /** @brief Required math function: ELU + * + * Requires a math function that calculates the element wise ELU of a tensor:\n + * @f[ + * result_{i} = \begin{cases} + \alpha \cdot (e^{x_i} - 1) & \text{if } x_i < 0 \\ + x_i & \text{if } x_i \geq 0 + \end{cases} + * @f] + * + * @param x N-dimensional tensor (input) + * @param result N-dimensional tensor (output) + */ + void (*elu)(const aitensor_t *x, const void *alpha, aitensor_t *result); + + /** @brief Required math function: Derivative of ELU + * + * Requires a math function that calculates the element wise ELU derivative of a tensor:\n + * @f[ + * result_{i} = \begin{cases} + \alpha \cdot e^{x_i} & \text{if } x_i < 0\\ + 1 & \text{if } x_i \geq 0 + \end{cases} + * @f] + * + * @param x N-dimensional tensor (input) + * @param result N-dimensional tensor (output) + */ + void (*d_elu)(const aitensor_t *x, const void *alpha, aitensor_t *result); + + /** @brief Required math function: Element wise tensor multiplication + * + * Requires a math function that multiplies two tensors element wise:\n + * @f[ + * result = a \circ b + * @f] + */ + void (*multiply)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + ///@} +}; + +/** @brief ELU layer type + * + * Defines the type of the layer (for example for type checks and debug prints). + * See aicore_layertype for more information about the layer type. + */ +extern const aicore_layertype_t *ailayer_elu_type; + +/** @brief Initialize and connect the given ELU layer + * + * This function represents the "constructor" of the abstract ELU layer. It initializes the layer structure + * and connects it to the previous layer.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example ailayer_elu_f32_default()). + * + * @param *layer The layer to initialize. + * @param *input_layer The previous layer that provides the inputs to the layer. + * @return Pointer to the (successfully) initialized general layer structure (ailayer_elu.base) + */ +ailayer_t *ailayer_elu(ailayer_elu_t *layer, ailayer_t *input_layer); + +/** @brief Calculate the forward pass for given ELU layer + * + * *Implementation of ailayer.forward.* + * + * It uses the result tensor of the previous layer as input and writes the result of the forward pass + * to the result tensor (ailayer.result) of the given layer. + * + * Calculation of the forward pass result: + * @f[ + * x_{out} \leftarrow ELU(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ x_{out} \f$: Result of the forward pass of this layer\n\n + * + * Used math functions: + * * ailayer_elu.elu + * + * @param *self Layer to calculate the forward path for. + */ +void ailayer_elu_forward(ailayer_t *self); + +/** @brief Calculate the backward pass for the given ELU layer + * + * *Implementation of ailayer.backward.* + * + * It uses the deltas tensor of the next layer as input and writes the result of the backward pass + * to the deltas tensor (ailayer.deltas) of the given layer. + * + * Calculation of the errors for the previous layer: + * @f[ + * \delta_{in} \leftarrow \delta_{out} \circ ELU'(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ \delta_{in} \f$: Result of the backward pass of this layer\n + * \f$ \delta_{out} \f$: Result of the backward pass of the next layer\n\n + * + * Used math functions: + * * ailayer_elu.elu + * * ailayer_elu.d_elu + * * ailayer_elu.multiply + * + * @param *self Layer to calculate the backward path for. + */ +void ailayer_elu_backward(ailayer_t *self); + +/** @brief Calculate the shape of the result tensor + * + * *Implementation of ailayer.calc_result_shape.* + * + * As the result tensor shape is shared with the result tensor shape of the previous layer (no change in shape is needed), + * this function returns without doing anything. + * + * @param *self Layer to calculate the resulting shape for. + */ +void ailayer_elu_calc_result_shape(ailayer_t *self); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the layer specification + * + * @param *self The layer to print the specification for + * @param *print Pointer to the print function to use + */ +void ailayer_elu_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // ELU_LAYER diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_input.c b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_input.c new file mode 100644 index 0000000..7c8213e --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_input.c @@ -0,0 +1,99 @@ +/** + * \file basic/base/ailayer/ailayer_input.c + * \version 2.0alpha + * \date 20.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/ailayer/ailayer_input.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_layertype_t ailayer_input_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "Input", + .print_specs = ailayer_input_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_layertype_t *ailayer_input_type = &ailayer_input_type_s; + + +ailayer_t *ailayer_input(ailayer_input_t *layer) +{ + layer->base.layer_type = ailayer_input_type; + + layer->base.layer_configuration = layer; + layer->base.result.dtype = layer->dtype; + layer->base.result.shape = layer->input_shape; + layer->base.result.dim = layer->input_dim; + + layer->base.deltas.dtype = 0; + + layer->base.forward = ailayer_input_forward; + layer->base.backward = ailayer_input_backward; + + layer->base.calc_result_shape = ailayer_input_calc_result_shape; + layer->base.sizeof_paramem = 0; + layer->base.set_paramem = 0; + layer->base.sizeof_trainmem = 0; + layer->base.set_trainmem = 0; + + layer->base.get_result_bound = 0; + + layer->base.trainable_params_count = 0; + + return &layer->base; +} + +void ailayer_input_forward(ailayer_t *self) +{ + return; +} + +void ailayer_input_backward(ailayer_t *self) +{ + return; +} + +void ailayer_input_calc_result_shape(ailayer_t *self) +{ + //ailayer_input_t *layer = (ailayer_input_t *)(self->layer_configuration); + + //self->result.shape[0] = configuration->input_shape[0]; + //self->result.shape[1] = configuration->input_shape[1]; + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void ailayer_input_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)) +{ + ailayer_input_t *self_casted = (ailayer_input_t *) self; + uint8_t i = 0; + + print("Dim: %d; Shape: [%d", self_casted->input_dim, self_casted->input_shape[i]); + for(i = 1; i < self_casted->input_dim; i++){ + print(", %d", self_casted->input_shape[i]); + } + print("]"); + return; +} +#endif diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_input.h b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_input.h new file mode 100644 index 0000000..66ed268 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_input.h @@ -0,0 +1,120 @@ +/** + * \file ailayer/ailayer_input.h + * \internal + * \date 20.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link ailayer layer \endlink implementation of the Input layer + * + * This is an "abstract" data-type independent implementation. To use the layer use one of the provided + * implementations for a specific hardware and data-type (for example from ailayer_input_default.h) or set + * the required math functions on your own. + * + * The Input layer is the start layer for every AIfES 2 model. + */ + +#ifndef INPUT_LAYER +#define INPUT_LAYER + +#include "core/aifes_core.h" + +typedef struct ailayer_input ailayer_input_t; + +/** @brief General \link ailayer_input.h Input layer \endlink structure +* +*/ +struct ailayer_input { + ailayer_t base; /**< Inherited field members from general ailayer struct. */ + const aimath_dtype_t *dtype; /**< Data type of the input and inference result values. */ + + /** @name Layer configuration + * @brief Required configuration parameters for the layer + * + * These fields have to be configured by the user before calling the initializer function. + */ + ///@{ + uint8_t input_dim; /**< Dimension of the input tensor. */ + uint16_t *input_shape; /**< Shape of the input tensor. */ + ///@} +}; + +/** @brief Input layer type + * + * Defines the type of the layer (for example for type checks and debug prints). + * See aicore_layertype for more information about the layer type. + */ +extern const aicore_layertype_t *ailayer_input_type; + +/** @brief Initialize the given Input layer + * + * This function represents the "constructor" of the abstract Input layer. It initializes the layer structure + * and connects it to the previous layer.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example ailayer_input_f32_default()). + * + * @param *layer The layer to initialize. + * @return Pointer to the (successfully) initialized general layer structure (ailayer_input.base). + */ +ailayer_t *ailayer_input(ailayer_input_t *layer); + +/** @brief Calculate the forward pass for given Input layer + * + * *Implementation of ailayer.forward.* + * + * The standard layer interface demands a forward pass function + * for the net scheduler, but theres no sense in 'calculating' + * an input layer, so this function returns without doing something. + * + * @param *self Layer to 'calculate' the forward pass for. + */ +void ailayer_input_forward(ailayer_t *self); + +/** @brief Calculate the backward pass for the given Input layer + * + * *Implementation of ailayer.backward.* + * + * The standard layer interface demands a backward pass function + * for the net scheduler, but theres no sense in 'calculating' + * an input layer, so this function returns without doing something. + * + * @param *self Layer to 'calculate' the backward pass for. + */ +void ailayer_input_backward(ailayer_t *self); + +/** @brief Calculate the shape of the result tensor + * + * *Implementation of ailayer.calc_result_shape* + * + * As the result shape is given by the configured ailayer_input.input_shape parameter, + * this function returns without doing anything. + * + * @param *self Layer to calculate the resulting shape for. + */ +void ailayer_input_calc_result_shape(ailayer_t *self); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the layer specification + * + * @param *self The layer to print the specification for + * @param *print Pointer to the print function to use + */ +void ailayer_input_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // INPUT_LAYER diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_leaky_relu.c b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_leaky_relu.c new file mode 100644 index 0000000..0f73b81 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_leaky_relu.c @@ -0,0 +1,112 @@ +/** + * \file basic/base/ailayer/ailayer_leaky_relu.c + * \version 2.0alpha + * \date 16.03.2021 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/ailayer/ailayer_leaky_relu.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_layertype_t ailayer_leaky_relu_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "Leaky ReLU", + .print_specs = ailayer_leaky_relu_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_layertype_t *ailayer_leaky_relu_type = &ailayer_leaky_relu_type_s; + +ailayer_t *ailayer_leaky_relu(ailayer_leaky_relu_t *layer, ailayer_t *input_layer) ////const void *beta1, +{ + layer->base.layer_type = ailayer_leaky_relu_type; + + layer->base.input_layer = input_layer; + input_layer->output_layer = &(layer->base); + layer->base.layer_configuration = layer; + layer->base.result.dtype = layer->dtype; + layer->base.result.shape = input_layer->result.shape; + layer->base.result.dim = input_layer->result.dim; + + layer->base.deltas.dtype = layer->dtype; + layer->base.deltas.dim = 2; + layer->base.deltas.shape = layer->base.result.shape; + + layer->base.forward = ailayer_leaky_relu_forward; + layer->base.backward = ailayer_leaky_relu_backward; + + layer->base.calc_result_shape = ailayer_leaky_relu_calc_result_shape; + layer->base.sizeof_paramem = 0; + layer->base.set_paramem = 0; + layer->base.sizeof_trainmem = 0; + layer->base.set_trainmem = 0; + + layer->base.trainable_params_count = 0; + + return &(layer->base); +} + +void ailayer_leaky_relu_forward(ailayer_t *self) +{ + ailayer_leaky_relu_t *layer = (ailayer_leaky_relu_t *)(self->layer_configuration); + aitensor_t *x_in = &(self->input_layer->result); + aitensor_t *x_out = &(self->result); + + layer->leaky_relu(x_in, layer->alpha, x_out); + return; +} + +void ailayer_leaky_relu_backward(ailayer_t *self) +{ + ailayer_leaky_relu_t *layer = (ailayer_leaky_relu_t *)(self->layer_configuration); + aitensor_t *delta_in = &(self->deltas); + aitensor_t *delta_out = &(self->output_layer->deltas); + aitensor_t *x_in = &(self->input_layer->result); + + // delta_in = delta_out .* relu'(x_in) + layer->d_leaky_relu(x_in, layer->alpha, delta_in); + layer->multiply(delta_in, delta_out, delta_in); + + return; +} + +void ailayer_leaky_relu_calc_result_shape(ailayer_t *self) +{ + /* Unused: Shape is already defined (Pointer) + + self->result.shape[0] = self->input_layer->result.shape[0]; + self->result.shape[1] = self->input_layer->result.shape[1]; + */ + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void ailayer_leaky_relu_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)) +{ + ailayer_leaky_relu_t *self_casted = (ailayer_leaky_relu_t *) self; + + print("alpha: "); + self_casted->dtype->print_aiscalar(self_casted->alpha, print); + return; +} +#endif + diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_leaky_relu.h b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_leaky_relu.h new file mode 100644 index 0000000..c012ea2 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_leaky_relu.h @@ -0,0 +1,200 @@ +/** + * \file basic/base/ailayer/ailayer_leaky_relu.h + * \internal + * \date 15.03.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link ailayer layer \endlink implementation of the Leaky ReLU activation layer + * + * This is an "abstract" data-type independent implementation. To use the layer use one of the provided + * implementations for a specific hardware and data-type (for example from ailayer_leaky_relu_default.h) or set + * the required math functions on your own. + * + * \image html ailayer_leaky_relu_schematic.png width=200px + * + * The Leaky ReLU layer is used as an activation function layer right after a dense layer. It calculates + * @f[ + * y = \begin{cases} + \alpha x & \text{if } x < 0\\ + x & \text{if } x \geq 0 + \end{cases} + * @f] + * for every element of the input tensor. + * + * The results of the forward pass of this layer are written to the result tensor of the base ailayer_t struct. + */ + +#ifndef LEAKY_RELU_LAYER +#define LEAKY_RELU_LAYER + +#include "core/aifes_core.h" + +typedef struct ailayer_leaky_relu ailayer_leaky_relu_t; + +/** @brief General \link ailayer_leaky_relu.h Leaky ReLU layer \endlink struct +* +*/ +struct ailayer_leaky_relu { + ailayer_t base; /**< Inherited field members from general ailayer struct. */ + const aimath_dtype_t *dtype; /**< Data type of the input and inference result values. */ + + /** @name Layer configuration + * @brief Required configuration parameters for the layer + * + * These fields have to be configured by the user before calling the initializer function. + */ + ///@{ + void *alpha; /**< Parameter \f$ \alpha \f$ used to calculate Leaky ReLU function for input values < 0. */ + ///@} + + /** @name Math functions + * @brief Required data type specific math functions + */ + ///@{ + + /** @brief Required math function: Leaky ReLU + * + * Requires a math function that calculates the element wise Leaky ReLU of a tensor:\n + * @f[ + * result_{i} = \begin{cases} + \alpha \cdot x_i & \text{if } x_i < 0 \\ + x_i & \text{if } x_i \geq 0 + \end{cases} + * @f] + * + * @param x N-dimensional tensor (input) + * @param result N-dimensional tensor (output) + */ + void (*leaky_relu)(const aitensor_t *x, const void *alpha, aitensor_t *result); + + /** @brief Required math function: Derivative of Leaky ReLU + * + * Requires a math function that calculates the element wise Leaky ReLU derivative of a tensor:\n + * @f[ + * result_{i} = \begin{cases} + \alpha & \text{if } x_i < 0\\ + 1 & \text{if } x_i \geq 0 + \end{cases} + * @f] + * + * @param x N-dimensional tensor (input) + * @param result N-dimensional tensor (output) + */ + void (*d_leaky_relu)(const aitensor_t *x, const void *alpha, aitensor_t *result); + + /** @brief Required math function: Element wise tensor multiplication + * + * Requires a math function that multiplies two tensors element wise:\n + * @f[ + * result = a \circ b + * @f] + */ + void (*multiply)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + ///@} +}; + +/** @brief Leaky ReLU layer type + * + * Defines the type of the layer (for example for type checks and debug prints). + * See aicore_layertype for more information about the layer type. + */ +extern const aicore_layertype_t *ailayer_leaky_relu_type; + +/** @brief Initialize and connect the given Leaky ReLU layer + * + * This function represents the "constructor" of the abstract Leaky ReLU layer. It initializes the layer structure + * and connects it to the previous layer.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example ailayer_leaky_relu_f32_default()). + * + * @param *layer The layer to initialize. + * @param *input_layer The previous layer that provides the inputs to the layer. + * @return Pointer to the (successfully) initialized general layer structure (ailayer_leaky_relu.base). + */ +ailayer_t *ailayer_leaky_relu(ailayer_leaky_relu_t *layer, ailayer_t *input_layer); + +/** @brief Calculate the forward pass for given Leaky ReLU layer + * + * *Implementation of ailayer.forward.* + * + * It uses the result tensor of the previous layer as input and writes the result of the forward pass + * to the result tensor (ailayer.result) of the given layer. + * + * Calculation of the forward pass result: + * @f[ + * x_{out} \leftarrow LeakyReLU(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ x_{out} \f$: Result of the forward pass of this layer\n\n + * + * Used math functions: + * * ailayer_leaky_relu.leaky_relu + * + * @param *self Layer to calculate the forward path for. + */ +void ailayer_leaky_relu_forward(ailayer_t *self); + +/** @brief Calculate the backward pass for the given Leaky ReLU layer + * + * *Implementation of ailayer.backward.* + * + * It uses the deltas tensor of the next layer as input and writes the result of the backward pass + * to the deltas tensor (ailayer.deltas) of the given layer. + * + * Calculation of the errors for the previous layer: + * @f[ + * \delta_{in} \leftarrow \delta_{out} \circ LeakyReLU'(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ \delta_{in} \f$: Result of the backward pass of this layer\n + * \f$ \delta_{out} \f$: Result of the backward pass of the next layer\n\n + * + * Used math functions: + * * ailayer_leaky_relu.leaky_relu + * * ailayer_leaky_relu.d_leaky_relu + * * ailayer_leaky_relu.multiply + * + * @param *self Layer to calculate the backward path for. + */ +void ailayer_leaky_relu_backward(ailayer_t *self); + +/** @brief Calculate the shape of the result tensor + * + * *Implementation of ailayer.calc_result_shape.* + * + * As the result tensor shape is shared with the result tensor shape of the previous layer (no change in shape is needed), + * this function returns without doing anything. + * + * @param *self Layer to calculate the resulting shape for. + */ +void ailayer_leaky_relu_calc_result_shape(ailayer_t *self); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the layer specification + * + * @param *self The layer to print the specification for + * @param *print Pointer to the print function to use + */ +void ailayer_leaky_relu_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // LEAKY_RELU_LAYER diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_relu.c b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_relu.c new file mode 100644 index 0000000..e252381 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_relu.c @@ -0,0 +1,110 @@ +/** + * \file basic/base/ailayer/ailayer_relu.c + * \version 2.0alpha + * \date 07.12.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/ailayer/ailayer_relu.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_layertype_t ailayer_relu_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "ReLU", + .print_specs = ailayer_relu_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_layertype_t *ailayer_relu_type = &ailayer_relu_type_s; + + +ailayer_t *ailayer_relu(ailayer_relu_t *layer, ailayer_t *input_layer) +{ + layer->base.layer_type = ailayer_relu_type; + + layer->base.input_layer = input_layer; + input_layer->output_layer = &(layer->base); + + layer->base.layer_configuration = layer; + layer->base.result.dtype = layer->dtype; + layer->base.result.shape = input_layer->result.shape; + layer->base.result.dim = input_layer->result.dim; + + layer->base.deltas.dtype = layer->dtype; + layer->base.deltas.dim = 2; + layer->base.deltas.shape = layer->base.result.shape; + + layer->base.forward = ailayer_relu_forward; + layer->base.backward = ailayer_relu_backward; + + layer->base.calc_result_shape = ailayer_relu_calc_result_shape; + layer->base.sizeof_paramem = 0; + layer->base.set_paramem = 0; + layer->base.sizeof_trainmem = 0; + layer->base.set_trainmem = 0; + + layer->base.trainable_params_count = 0; + + return &(layer->base); +} + +void ailayer_relu_forward(ailayer_t *self) +{ + ailayer_relu_t *layer = (ailayer_relu_t *)(self->layer_configuration); + aitensor_t *x_in = &(self->input_layer->result); + aitensor_t *x_out = &(self->result); + + layer->relu(x_in, x_out); + return; +} + + +void ailayer_relu_backward(ailayer_t *self) +{ + ailayer_relu_t *layer = (ailayer_relu_t *)(self->layer_configuration); + aitensor_t *delta_in = &(self->deltas); + aitensor_t *delta_out = &(self->output_layer->deltas); + aitensor_t *x_in = &(self->input_layer->result); + + // delta_in = delta_out .* relu'(x_in) + layer->d_relu(x_in, delta_in); + layer->multiply(delta_in, delta_out, delta_in); + return; +} + +void ailayer_relu_calc_result_shape(ailayer_t *self) +{ + /* Unused: Shape is already defined (Pointer) + + self->result.shape[0] = self->input_layer->result.shape[0]; + self->result.shape[1] = self->input_layer->result.shape[1]; + */ + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void ailayer_relu_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)) +{ + return; +} +#endif + diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_relu.h b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_relu.h new file mode 100644 index 0000000..469aeaf --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_relu.h @@ -0,0 +1,187 @@ + +/** + * \file basic/base/ailayer/ailayer_relu.h + * \internal + * \date 07.12.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link ailayer layer \endlink implementation of the ReLU activation layer + * + * This is an "abstract" data-type independent implementation. To use the layer use one of the provided + * implementations for a specific hardware and data-type (for example from ailayer_relu_default.h) or set + * the required math functions on your own. + * + * \image html ailayer_relu_schematic.png width=200px + * + * The ReLU layer is used as an activation function layer right after a dense layer. It calculates + * @f[ + * y = \begin{cases} + 0 & \text{if } x < 0\\ + x & \text{if } x \geq 0 + \end{cases} + * @f] + * for every element of the input tensor. + * + * The results of the forward pass of this layer are written to the result tensor of the base ailayer_t struct. + */ + +#ifndef RELU_LAYER +#define RELU_LAYER + +#include "core/aifes_core.h" + +typedef struct ailayer_relu ailayer_relu_t; + +/** @brief General \link ailayer_relu.h ReLU layer \endlink struct +* +*/ +struct ailayer_relu { + ailayer_t base; /**< Inherited field members from general ailayer struct. */ + const aimath_dtype_t *dtype; /**< Data type of the input and inference result values. */ + + /** @name Math functions + * @brief Required data type specific math functions + */ + ///@{ + + /** @brief Required math function: ReLU + * + * Requires a math function that calculates the element wise ReLU of a tensor:\n + * @f[ + * result_{i} = max(0, x_{i}) + * @f] + * + * @param x N-dimensional tensor (input) + * @param result N-dimensional tensor (output) + */ + void (*relu)(const aitensor_t *x, aitensor_t *result); + + /** @brief Required math function: Derivative of ReLU + * + * Requires a math function that calculates the element wise ReLU derivative of a tensor:\n + * @f[ + * result_{i} = \begin{cases} + 0 & \text{if } x_i < 0\\ + 1 & \text{if } x_i \geq 0 + \end{cases} + * @f] + * + * @param x N-dimensional tensor (input) + * @param result N-dimensional tensor (output) + */ + void (*d_relu)(const aitensor_t *x, aitensor_t *result); + + /** @brief Required math function: Element wise tensor multiplication + * + * Requires a math function that multiplies two tensors element wise:\n + * @f[ + * result = a \circ b + * @f] + */ + void (*multiply)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); +}; + +/** @brief ReLU layer type + * + * Defines the type of the layer (for example for type checks and debug prints). + * See aicore_layertype for more information about the layer type. + */ +extern const aicore_layertype_t *ailayer_relu_type; + +/** @brief Initialize and connect the given ReLU layer + * + * This function represents the "constructor" of the abstract ReLU layer. It initializes the layer structure + * and connects it to the previous layer.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example ailayer_relu_f32_default()). + * + * @param *layer The layer to initialize. + * @param *input_layer The previous layer that provides the inputs to the layer. + * @return Pointer to the (successfully) initialized general layer structure (ailayer_relu.base). + */ +ailayer_t *ailayer_relu(ailayer_relu_t *layer, ailayer_t *input_layer); + +/** @brief Calculate the forward pass for given ReLU layer + * + * *Implementation of ailayer.forward.* + * + * It uses the result tensor of the previous layer as input and writes the result of the forward pass + * to the result tensor (ailayer.result) of the given layer. + * + * Calculation of the forward pass result: + * @f[ + * x_{out} \leftarrow ReLU(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ x_{out} \f$: Result of the forward pass of this layer\n\n + * + * Used math functions: + * * ailayer_relu.relu + * + * @param *self Layer to calculate the forward path for. + */ +void ailayer_relu_forward(ailayer_t *self); + +/** @brief Calculate the backward pass for the given ReLU layer + * + * *Implementation of ailayer.backward.* + * + * It uses the deltas tensor of the next layer as input and writes the result of the backward pass + * to the deltas tensor (ailayer.deltas) of the given layer. + * + * Calculation of the errors for the previous layer: + * @f[ + * \delta_{in} \leftarrow \delta_{out} \circ ReLU'(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ \delta_{in} \f$: Result of the backward pass of this layer\n + * \f$ \delta_{out} \f$: Result of the backward pass of the next layer\n\n + * + * Used math functions: + * * ailayer_relu.relu + * * ailayer_relu.d_relu + * * ailayer_relu.multiply + * + * @param *self Layer to calculate the backward path for. + */ +void ailayer_relu_backward(ailayer_t *self); + +/** @brief Calculate the shape of the result tensor + * + * *Implementation of ailayer.calc_result_shape.* + * + * As the result tensor shape is shared with the result tensor shape of the previous layer (no change in shape is needed), + * this function returns without doing anything. + * + * @param *self Layer to calculate the resulting shape for. + */ +void ailayer_relu_calc_result_shape(ailayer_t *self); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the layer specification + * + * @param *self The layer to print the specification for + * @param *print Pointer to the print function to use + */ +void ailayer_relu_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // RELU_LAYER diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_sigmoid.c b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_sigmoid.c new file mode 100644 index 0000000..21b9252 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_sigmoid.c @@ -0,0 +1,122 @@ +/** + * \file basic/base/ailayer/ailayer_sigmoid.c + * \version 2.0alpha + * \date 20.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/ailayer/ailayer_sigmoid.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_layertype_t ailayer_sigmoid_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "Sigmoid", + .print_specs = ailayer_sigmoid_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_layertype_t *ailayer_sigmoid_type = &ailayer_sigmoid_type_s; + + +ailayer_t *ailayer_sigmoid(ailayer_sigmoid_t *layer, ailayer_t *input_layer) +{ + layer->base.layer_type = ailayer_sigmoid_type; + + layer->base.input_layer = input_layer; + input_layer->output_layer = &(layer->base); + + layer->base.layer_configuration = layer; + layer->base.result.dtype = layer->dtype; + layer->base.result.dim = input_layer->result.dim; + layer->base.result.shape = input_layer->result.shape; + + layer->base.deltas.dtype = layer->dtype; + layer->base.deltas.dim = 2; + layer->base.deltas.shape = layer->base.result.shape; + + layer->base.forward = ailayer_sigmoid_forward; + layer->base.backward = ailayer_sigmoid_backward; + + layer->base.calc_result_shape = ailayer_sigmoid_calc_result_shape; + layer->base.sizeof_paramem = 0; + layer->base.set_paramem = 0; + layer->base.sizeof_trainmem = 0; + layer->base.set_trainmem = 0; + + layer->base.trainable_params_count = 0; + + return &(layer->base); +} + +void ailayer_sigmoid_forward(ailayer_t *self) +{ + ailayer_sigmoid_t *layer = (ailayer_sigmoid_t *)(self->layer_configuration); + aitensor_t *x_in = &(self->input_layer->result); + aitensor_t *x_out = &(self->result); + + layer->sigmoid(x_in, x_out); + return; +} + + +void ailayer_sigmoid_backward(ailayer_t *self) +{ + ailayer_sigmoid_t *layer = (ailayer_sigmoid_t *)(self->layer_configuration); + aitensor_t *delta_in = &(self->deltas); + aitensor_t *delta_out = &(self->output_layer->deltas); + aitensor_t *x_in = &(self->input_layer->result); + + int8_t temp_result_params[aimath_sizeof_tensor_params(x_in) + 1]; // +1 to prevent array of size 0 + int8_t temp_result_data[aimath_sizeof_tensor_data(x_in)]; + aitensor_t temp_result = { + .dim = 2, + .shape = x_in->shape, + .data = temp_result_data, + .dtype = x_in->dtype, + .tensor_params = temp_result_params + }; + + // delta_in = delta_out .* sigmoid'(x_in) + layer->sigmoid(x_in, &temp_result); + layer->d_sigmoid(&temp_result, &temp_result); + layer->multiply(&temp_result, delta_out, delta_in); + + return; +} + +void ailayer_sigmoid_calc_result_shape(ailayer_t *self) +{ + /* Unused: Shape is already defined (Pointer) + + self->result.shape[0] = self->input_layer->result.shape[0]; + self->result.shape[1] = self->input_layer->result.shape[1]; + */ + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void ailayer_sigmoid_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)) +{ + return; +} +#endif + diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_sigmoid.h b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_sigmoid.h new file mode 100644 index 0000000..1683f96 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_sigmoid.h @@ -0,0 +1,182 @@ +/** + * \file basic/base/ailayer/ailayer_sigmoid.h + * \internal + * \date 20.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link ailayer layer \endlink implementation of the Sigmoid activation layer + * + * This is an "abstract" data-type independent implementation. To use the layer use one of the provided + * implementations for a specific hardware and data-type (for example from ailayer_sigmoid_default.h) or set + * the required math functions on your own. + * + * \image html ailayer_sigmoid_schematic.png width=200px + * + * The Sigmoid layer is used as an activation function layer right after a dense layer. It calculates + * @f[ + * y = \sigma(x) = \frac{1}{1+e^{-x}} + * @f] + * for every element of the input tensor. + * + * The results of the forward pass of this layer are written to the result tensor of the base ailayer_t struct. + */ + +#ifndef SIGMOID_LAYER +#define SIGMOID_LAYER + +#include "core/aifes_core.h" + +typedef struct ailayer_sigmoid ailayer_sigmoid_t; + +/** @brief General \link ailayer_sigmoid.h Sigmoid layer \endlink struct +* +*/ +struct ailayer_sigmoid { + ailayer_t base; /**< Inherited field members from general ailayer struct. */ + const aimath_dtype_t *dtype; /**< Data type of the input and inference result values. */ + + /** @name Math functions + * @brief Required data type specific math functions + */ + ///@{ + + /** @brief Required math function: Sigmoid + * + * Requires a math function that calculates the element wise sigmoid of a tensor:\n + * @f[ + * result_{i} = \sigma(x_{i}) = \frac{1}{1 + e^{-x_{i}}} + * @f] + * + * @param x N-dimensional tensor (input) + * @param result N-dimensional tensor (output) + */ + void (*sigmoid)(const aitensor_t *x, aitensor_t *result); + + /** @brief Required math function: Derivative of sigmoid + * + * Requires a math function that calculates the element wise sigmoid derivative of a tensor:\n + * @f[ + * result_{i} = \sigma'(x_{i}) = \sigma(x_{i}) \cdot (1 - \sigma(x_{i})) + * @f] + * + * @param sigmoid_x N-dimensional tensor with the sigmoid values \f$ \sigma(x_{i}) \f$ (input) + * @param result N-dimensional tensor (output) + */ + void (*d_sigmoid)(const aitensor_t *sigmoid_x, aitensor_t *result); + + /** @brief Required math function: Element wise tensor multiplication + * + * Requires a math function that multiplies two tensors element wise:\n + * @f[ + * result = a \circ b + * @f] + */ + void (*multiply)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + ///@} +}; + +/** @brief Sigmoid layer type + * + * Defines the type of the layer (for example for type checks and debug prints). + * See aicore_layertype for more information about the layer type. + */ +extern const aicore_layertype_t *ailayer_sigmoid_type; + +/** @brief Initialize and connect the given Sigmoid layer + * + * This function represents the "constructor" of the abstract Sigmoid layer. It initializes the layer structure + * and connects it to the previous layer.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example ailayer_sigmoid_f32_default()). + * + * @param *layer The layer to initialize. + * @param *input_layer The previous layer that provides the inputs to the layer. + * @return Pointer to the (successfully) initialized general layer structure (ailayer_sigmoid.base). + */ +ailayer_t *ailayer_sigmoid(ailayer_sigmoid_t *layer, ailayer_t *input_layer); + +/** @brief Calculate the forward pass for given Sigmoid layer + * + * *Implementation of ailayer.forward.* + * + * It uses the result tensor of the previous layer as input and writes the result of the forward pass + * to the result tensor (ailayer.result) of the given layer. + * + * Calculation of the forward pass result: + * @f[ + * x_{out} \leftarrow \sigma(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ x_{out} \f$: Result of the forward pass of this layer\n\n + * + * Used math functions: + * * ailayer_sigmoid.sigmoid + * + * @param *self Layer to calculate the forward path for. + */ +void ailayer_sigmoid_forward(ailayer_t *self); + +/** @brief Calculate the backward pass for the given Sigmoid layer + * + * *Implementation of ailayer.backward.* + * + * It uses the deltas tensor of the next layer as input and writes the result of the backward pass + * to the deltas tensor (ailayer.deltas) of the given layer. + * + * Calculation of the errors for the previous layer: + * @f[ + * \delta_{in} \leftarrow \delta_{out} \circ \sigma'(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ \delta_{in} \f$: Result of the backward pass of this layer\n + * \f$ \delta_{out} \f$: Result of the backward pass of the next layer\n\n + * + * Used math functions: + * * ailayer_sigmoid.sigmoid + * * ailayer_sigmoid.d_sigmoid + * * ailayer_sigmoid.multiply + * + * @param *self Layer to calculate the backward path for. + */ +void ailayer_sigmoid_backward(ailayer_t *self); + +/** @brief Calculate the shape of the result tensor + * + * *Implementation of ailayer.calc_result_shape.* + * + * As the result tensor shape is shared with the result tensor shape of the previous layer (no change in shape is needed), + * this function returns without doing anything. + * + * @param *self Layer to calculate the resulting shape for. + */ +void ailayer_sigmoid_calc_result_shape(ailayer_t *self); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the layer specification + * + * @param *self The layer to print the specification for + * @param *print Pointer to the print function to use + */ +void ailayer_sigmoid_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // SIGMOID_LAYER diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_softmax.c b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_softmax.c new file mode 100644 index 0000000..f7d2ae4 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_softmax.c @@ -0,0 +1,95 @@ +/** + * \file basic/base/ailayer/ailayer_softmax.c + * \version 2.0alpha + * \date 07.12.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/ailayer/ailayer_softmax.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_layertype_t ailayer_softmax_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "Softmax", + .print_specs = ailayer_softmax_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_layertype_t *ailayer_softmax_type = &ailayer_softmax_type_s; + + +ailayer_t *ailayer_softmax(ailayer_softmax_t *layer, ailayer_t *input_layer) +{ + layer->base.layer_type = ailayer_softmax_type; + + layer->base.input_layer = input_layer; + input_layer->output_layer = &(layer->base); + + layer->base.layer_configuration = layer; + layer->base.result.dtype = layer->dtype; + layer->base.result.shape = input_layer->result.shape; + layer->base.result.dim = input_layer->result.dim; + + // No backward pass supported yet + layer->base.deltas.dtype = 0; + + layer->base.forward = ailayer_softmax_forward; + layer->base.backward = 0; + + layer->base.calc_result_shape = ailayer_softmax_calc_result_shape; + layer->base.sizeof_paramem = 0; + layer->base.set_paramem = 0; + layer->base.sizeof_trainmem = 0; + layer->base.set_trainmem = 0; + + layer->base.trainable_params_count = 0; + + return &(layer->base); +} + +void ailayer_softmax_forward(ailayer_t *self) +{ + ailayer_softmax_t *layer = (ailayer_softmax_t *)(self->layer_configuration); + aitensor_t *x_in = &(self->input_layer->result); + aitensor_t *x_out = &(self->result); + + layer->softmax(x_in, x_out); + return; +} + +void ailayer_softmax_calc_result_shape(ailayer_t *self) +{ + /* Unused: Shape is already defined (Pointer) + + self->result.shape[0] = self->input_layer->result.shape[0]; + self->result.shape[1] = self->input_layer->result.shape[1]; + */ + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void ailayer_softmax_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)) +{ + return; +} +#endif + diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_softmax.h b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_softmax.h new file mode 100644 index 0000000..9b97ec2 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_softmax.h @@ -0,0 +1,140 @@ + +/** + * \file basic/base/ailayer/ailayer_softmax.h + * \internal + * \date 07.12.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link ailayer layer \endlink implementation of the Softmax activation layer + * + * This is an "abstract" data-type independent implementation. To use the layer use one of the provided + * implementations for a specific hardware and data-type (for example from ailayer_softmax_default.h) or set + * the required math functions on your own.\n + * This layer is designed as an output-layer and has no backward implementation. If you want to train an ANN with this layer, + * use the \link ailoss_crossentropy.h Cross-Entropy loss \endlink which has a combined Softmax-Cross-Entropy backward + * function that is faster to compute! + * + * \image html ailayer_softmax_schematic.png width=200px + * + * The Softmax layer is used as an activation function layer right after a dense layer. It calculates + * @f[ + * y_i = \frac{e^{x_i}}{\sum_{j=1}^{K} e^{x_j}} + * @f] + * for every element of the input tensor with \f$ K \f$ elements. + * + * The results of the forward pass of this layer are written to the result tensor of the base ailayer_t struct. + */ + +#ifndef SOFTMAX_LAYER +#define SOFTMAX_LAYER + +#include "core/aifes_core.h" + +typedef struct ailayer_softmax ailayer_softmax_t; + +/** @brief General \link ailayer_softmax.h Softmax layer \endlink struct +* +*/ +struct ailayer_softmax { + ailayer_t base; /**< Inherited field members from general ailayer struct. */ + const aimath_dtype_t *dtype; /**< Data type of the input and inference result values. */ + + /** @name Math functions + * @brief Required data type specific math functions + */ + ///@{ + + /** @brief Required math function: Softmax + * + * Requires a math function that calculates the element wise softmax of a tensor:\n + * @f[ + * result_{i} = \frac{e^{x_i}}{\sum_{j=1}^{K} e^{x_j}} + * @f] + * + * @param x N-dimensional tensor (input) + * @param result N-dimensional tensor (output) + */ + void (*softmax)(const aitensor_t *x, aitensor_t *result); + + ///@} +}; + +/** @brief Softmax layer type + * + * Defines the type of the layer (for example for type checks and debug prints). + * See aicore_layertype for more information about the layer type. + */ +extern const aicore_layertype_t *ailayer_softmax_type; + +/** @brief Initialize and connect the given Softmax layer + * + * This function represents the "constructor" of the abstract Softmax layer. It initializes the layer structure + * and connects it to the previous layer.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example ailayer_softmax_f32_default()). + * + * @param *layer The layer to initialize. + * @param *input_layer The previous layer that provides the inputs to the layer. + * @return Pointer to the (successfully) initialized general layer structure (ailayer_softmax.base). + */ +ailayer_t *ailayer_softmax(ailayer_softmax_t *layer, ailayer_t *input_layer); + +/** @brief Calculate the forward pass for given Softmax layer + * + * *Implementation of ailayer.forward.* + * + * It uses the result tensor of the previous layer as input and writes the result of the forward pass + * to the result tensor (ailayer.result) of the given layer. + * + * Calculation of the forward pass result: + * @f[ + * x_{out} \leftarrow Softmax(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ x_{out} \f$: Result of the forward pass of this layer\n\n + * + * Used math functions: + * * ailayer_softmax.softmax + * + * @param *self Layer to calculate the forward path for. + */ +void ailayer_softmax_forward(ailayer_t *self); + +/** @brief Calculate the shape of the result tensor + * + * *Implementation of ailayer.calc_result_shape.* + * + * As the result tensor shape is shared with the result tensor shape of the previous layer (no change in shape is needed), + * this function returns without doing anything. + * + * @param *self Layer to calculate the resulting shape for. + */ +void ailayer_softmax_calc_result_shape(ailayer_t *self); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the layer specification + * + * @param *self The layer to print the specification for + * @param *print Pointer to the print function to use + */ +void ailayer_softmax_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // SOFTMAX_LAYER diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_softsign.c b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_softsign.c new file mode 100644 index 0000000..9cc1152 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_softsign.c @@ -0,0 +1,120 @@ +/** + * \file basic/base/ailayer/ailayer_softsign.c + * \version 2.0alpha + * \date 16.03.2021 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief A softsign activation function. + * \details + */ + +#include "basic/base/ailayer/ailayer_softsign.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_layertype_t ailayer_softsign_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "Softsign", + .print_specs = ailayer_softsign_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_layertype_t *ailayer_softsign_type = &ailayer_softsign_type_s; + + +ailayer_t *ailayer_softsign(ailayer_softsign_t *layer, ailayer_t *input_layer) +{ + layer->base.layer_type = ailayer_softsign_type; + + layer->base.input_layer = input_layer; + input_layer->output_layer = &(layer->base); + + layer->base.layer_configuration = layer; + layer->base.result.dtype = layer->dtype; + + layer->base.result.dim = input_layer->result.dim; + layer->base.result.shape = input_layer->result.shape; + + layer->base.deltas.dtype = layer->dtype; + layer->base.deltas.dim = 2; + layer->base.deltas.shape = layer->base.result.shape; + + layer->base.forward = ailayer_softsign_forward; + layer->base.backward = ailayer_softsign_backward; + + layer->base.calc_result_shape = ailayer_softsign_calc_result_shape; + layer->base.sizeof_paramem = 0; + layer->base.set_paramem = 0; + layer->base.sizeof_trainmem = 0; + layer->base.set_trainmem = 0; + + layer->base.trainable_params_count = 0; + + return &(layer->base); +} + +void ailayer_softsign_backward(ailayer_t *self) +{ + ailayer_softsign_t *layer = (ailayer_softsign_t *)(self->layer_configuration); + aitensor_t *delta_in = &(self->deltas); + aitensor_t *delta_out = &(self->output_layer->deltas); + aitensor_t *x_in = &(self->input_layer->result); + + int8_t temp_result_params[aimath_sizeof_tensor_params(x_in) + 1]; // +1 to prevent array of size 0 + int8_t temp_result_data[aimath_sizeof_tensor_data(x_in)]; + aitensor_t temp_result = { + .dim = 2, + .shape = x_in->shape, + .data = temp_result_data, + .dtype = x_in->dtype, + .tensor_params = temp_result_params + }; + + layer->softsign(x_in, &temp_result); + layer->d_softsign(&temp_result, &temp_result); + layer->multiply(&temp_result, delta_out, delta_in); + + return; +} + +void ailayer_softsign_forward(ailayer_t *self) +{ + ailayer_softsign_t *layer = (ailayer_softsign_t *)(self->layer_configuration); + aitensor_t *x_in = &(self->input_layer->result); + aitensor_t *x_out = &(self->result); + + layer->softsign(x_in, x_out); + return; +} + +void ailayer_softsign_calc_result_shape(ailayer_t *self) +{ + /* Unused: Shape is already defined (Pointer) + + self->result.shape[0] = self->input_layer->result.shape[0]; + self->result.shape[1] = self->input_layer->result.shape[1]; + */ + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void ailayer_softsign_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)) +{ + return; +} +#endif diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_softsign.h b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_softsign.h new file mode 100644 index 0000000..01524b8 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_softsign.h @@ -0,0 +1,183 @@ + +/** + * \file basic/base/ailayer/ailayer_softsign.h + * \internal + * \date 16.03.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link ailayer layer \endlink implementation of the Softsign activation layer + * + * This is an "abstract" data-type independent implementation. To use the layer use one of the provided + * implementations for a specific hardware and data-type (for example from ailayer_softsign_default.h) or set + * the required math functions on your own. + * + * \image html ailayer_softsign_schematic.png width=200px + * + * The Softsign layer is used as an activation function layer right after a dense layer. It calculates + * @f[ + * y = \frac {x} {1 + |x|} + * @f] + * for every element of the input tensor. + * + * The results of the forward pass of this layer are written to the result tensor of the base ailayer_t struct. + */ + +#ifndef SOFTSIGN_LAYER +#define SOFTSIGN_LAYER + +#include "core/aifes_core.h" + +typedef struct ailayer_softsign ailayer_softsign_t; + +/** @brief General \link ailayer_softsign.h Softsign layer \endlink struct +* +*/ +struct ailayer_softsign { + ailayer_t base; /**< Inherited field members from general ailayer struct. */ + const aimath_dtype_t *dtype; /**< Data type of the input and inference result values. */ + + /** @name Math functions + * @brief Required data type specific math functions + */ + ///@{ + + /** @brief Required math function: Softsign + * + * Requires a math function that calculates the element wise softsign of a tensor:\n + * @f[ + * result_{i} = \frac {x_i} {1 + |x_i|} + * @f] + * + * @param x N-dimensional tensor (input) + * @param result N-dimensional tensor (output) + */ + void (*softsign)(const aitensor_t *x, aitensor_t *result); + + /** @brief Required math function: Derivative of softsign + * + * Requires a math function that calculates the element wise softsign derivative of a tensor:\n + * @f[ + * result_{i} = \frac {x_i} {(1 + |x_i|)^2} + * @f] + * + * @param x N-dimensional tensor (input) + * @param result N-dimensional tensor (output) + */ + void (*d_softsign)(const aitensor_t *softsign_x, aitensor_t *result); + + /** @brief Required math function: Element wise tensor multiplication + * + * Requires a math function that multiplies two tensors element wise:\n + * @f[ + * result = a \circ b + * @f] + */ + void (*multiply)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + ///@} +}; + +/** @brief Softsign layer type + * + * Defines the type of the layer (for example for type checks and debug prints). + * See aicore_layertype for more information about the layer type. + */ +extern const aicore_layertype_t *ailayer_softsign_type; + +/** @brief Initialize and connect the given Softsign layer + * + * This function represents the "constructor" of the abstract Softsign layer. It initializes the layer structure + * and connects it to the previous layer.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example ailayer_softsign_f32_default()). + * + * @param *layer The layer to initialize. + * @param *input_layer The previous layer that provides the inputs to the layer. + * @return Pointer to the (successfully) initialized general layer structure (ailayer_softsign.base). + */ +ailayer_t *ailayer_softsign(ailayer_softsign_t *layer, ailayer_t *input_layer); + +/** @brief Calculate the forward pass for given Tanh layer + * + * *Implementation of ailayer.forward.* + * + * It uses the result tensor of the previous layer as input and writes the result of the forward pass + * to the result tensor (ailayer.result) of the given layer. + * + * Calculation of the forward pass result: + * @f[ + * x_{out} \leftarrow Softsign(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ x_{out} \f$: Result of the forward pass of this layer\n\n + * + * Used math functions: + * * ailayer_softsign.softsign + * + * @param *self Layer to calculate the forward pass for. + */ +void ailayer_softsign_forward(ailayer_t *self); + +/** @brief Calculate the backward pass for the given Softsign layer + * + * *Implementation of ailayer.backward.* + * + * It uses the deltas tensor of the next layer as input and writes the result of the backward pass + * to the deltas tensor (ailayer.deltas) of the given layer. + * + * Calculation of the errors for the previous layer: + * @f[ + * \delta_{in} \leftarrow \delta_{out} \circ Softsign'(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ \delta_{in} \f$: Result of the backward pass of this layer\n + * \f$ \delta_{out} \f$: Result of the backward pass of the next layer\n\n + * + * Used math functions: + * * ailayer_softsign.softsign + * * ailayer_softsign.d_softsign + * * ailayer_softsign.multiply + * + * @param *self Layer to calculate the backward path for. + */ +void ailayer_softsign_backward(ailayer_t *self); + +/** @brief Calculate the shape of the result tensor + * + * *Implementation of ailayer.calc_result_shape.* + * + * As the result tensor shape is shared with the result tensor shape of the previous layer (no change in shape is needed), + * this function returns without doing anything. + * + * @param *self Layer to calculate the resulting shape for. + */ +void ailayer_softsign_calc_result_shape(ailayer_t *self); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the layer specification + * + * @param *self The layer to print the specification for + * @param *print Pointer to the print function to use + */ +void ailayer_softsign_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // SOFTSIGN_LAYER diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_tanh.c b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_tanh.c new file mode 100644 index 0000000..ed88e2b --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_tanh.c @@ -0,0 +1,123 @@ + +/** + * \file basic/base/ailayer/ailayer_tanh.c + * \version 2.0alpha + * \date 17.03.2021 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/ailayer/ailayer_tanh.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_layertype_t ailayer_tanh_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "Tanh", + .print_specs = ailayer_tanh_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_layertype_t *ailayer_tanh_type = &ailayer_tanh_type_s; + + +ailayer_t *ailayer_tanh(ailayer_tanh_t *layer, ailayer_t *input_layer) +{ + layer->base.layer_type = ailayer_tanh_type; + + layer->base.input_layer = input_layer; + input_layer->output_layer = &(layer->base); + + layer->base.layer_configuration = layer; + layer->base.result.dtype = layer->dtype; + layer->base.result.dim = input_layer->result.dim; + layer->base.result.shape = input_layer->result.shape; + + layer->base.deltas.dtype = layer->dtype; + layer->base.deltas.dim = 2; + layer->base.deltas.shape = layer->base.result.shape; + + layer->base.forward = ailayer_tanh_forward; + layer->base.backward = ailayer_tanh_backward; + + layer->base.calc_result_shape = ailayer_tanh_calc_result_shape; + layer->base.sizeof_paramem = 0; + layer->base.set_paramem = 0; + layer->base.sizeof_trainmem = 0; + layer->base.set_trainmem = 0; + + layer->base.trainable_params_count = 0; + + return &(layer->base); +} + +void ailayer_tanh_forward(ailayer_t *self) +{ + ailayer_tanh_t *layer = (ailayer_tanh_t *)(self->layer_configuration); + aitensor_t *x_in = &(self->input_layer->result); + aitensor_t *x_out = &(self->result); + + layer->tanh(x_in, x_out); + return; +} + + +void ailayer_tanh_backward(ailayer_t *self) +{ + ailayer_tanh_t *layer = (ailayer_tanh_t *)(self->layer_configuration); + aitensor_t *delta_in = &(self->deltas); + aitensor_t *delta_out = &(self->output_layer->deltas); + aitensor_t *x_in = &(self->input_layer->result); + + int8_t temp_result_params[aimath_sizeof_tensor_params(x_in) + 1]; // +1 to prevent array of size 0 + int8_t temp_result_data[aimath_sizeof_tensor_data(x_in)]; + aitensor_t temp_result = { + .dim = 2, + .shape = x_in->shape, + .data = temp_result_data, + .dtype = x_in->dtype, + .tensor_params = temp_result_params + }; + + // delta_in = delta_out .* tanh'(x_in) + layer->tanh(x_in, &temp_result); + layer->d_tanh(&temp_result, &temp_result); + layer->multiply(&temp_result, delta_out, delta_in); + + return; +} + +void ailayer_tanh_calc_result_shape(ailayer_t *self) +{ + /* Unused: Shape is already defined (Pointer) + + self->result.shape[0] = self->input_layer->result.shape[0]; + self->result.shape[1] = self->input_layer->result.shape[1]; + */ + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void ailayer_tanh_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)) +{ + return; +} +#endif + diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_tanh.h b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_tanh.h new file mode 100644 index 0000000..177aef5 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_tanh.h @@ -0,0 +1,182 @@ + +/** + * \file basic/base/ailayer/ailayer_tanh.h + * \internal + * \date 17.03.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link ailayer layer \endlink implementation of the Tanh activation layer + * + * This is an "abstract" data-type independent implementation. To use the layer use one of the provided + * implementations for a specific hardware and data-type (for example from ailayer_tanh_default.h) or set + * the required math functions on your own. + * + * \image html ailayer_tanh_schematic.png width=200px + * + * The Tanh layer is used as an activation function layer right after a dense layer. It calculates + * @f[ + * y = \tanh(x) = \frac{e^x - e^{-x}}{e^x + e^{-x}} + * @f] + * for every element of the input tensor. + * + * The results of the forward pass of this layer are written to the result tensor of the base ailayer_t struct. + */ +#ifndef TANH_LAYER +#define TANH_LAYER + +#include "core/aifes_core.h" + +typedef struct ailayer_tanh ailayer_tanh_t; + +/** @brief General \link ailayer_tanh.h Tanh layer \endlink struct +* +*/ +struct ailayer_tanh { + ailayer_t base; /**< Inherited field members from general ailayer struct. */ + const aimath_dtype_t *dtype; /**< Data type of the input and inference result values. */ + + /** @name Math functions + * @brief Required data type specific math functions + */ + ///@{ + + /** @brief Required math function: Tanh + * + * Requires a math function that calculates the element wise tanh of a tensor:\n + * @f[ + * result_{i} = \tanh(x_{i}) = \frac{e^{x_i} - e^{-x_i}}{e^{x_i} + e^{-x_i}} + * @f] + * + * @param x N-dimensional tensor (input) + * @param result N-dimensional tensor (output) + */ + void (*tanh)(const aitensor_t *x, aitensor_t *result); + + /** @brief Required math function: Derivative of tanh + * + * Requires a math function that calculates the element wise tanh derivative of a tensor:\n + * @f[ + * result_{i} = tanh'(x_{i}) = 1 - tanh(x_{i})^2 + * @f] + * + * @param tanh_x N-dimensional tensor with the tanh values \f$ tanh(x_{i}) \f$ (input) + * @param result N-dimensional tensor (output) + */ + void (*d_tanh)(const aitensor_t *tanh_x, aitensor_t *result); + + /** @brief Required math function: Element wise tensor multiplication + * + * Requires a math function that multiplies two tensors element wise:\n + * @f[ + * result = a \circ b + * @f] + */ + void (*multiply)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + ///@} +}; + +/** @brief Tanh layer type + * + * Defines the type of the layer (for example for type checks and debug prints). + * See aicore_layertype for more information about the layer type. + */ +extern const aicore_layertype_t *ailayer_tanh_type; + +/** @brief Initialize and connect the given Tanh layer + * + * This function represents the "constructor" of the abstract Tanh layer. It initializes the layer structure + * and connects it to the previous layer.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example ailayer_tanh_f32_default()). + * + * @param *layer The layer to initialize. + * @param *input_layer The previous layer that provides the inputs to the layer. + * @return Pointer to the (successfully) initialized general layer structure (ailayer_tanh.base). + */ +ailayer_t *ailayer_tanh(ailayer_tanh_t *layer, ailayer_t *input_layer); + +/** @brief Calculate the forward pass for given Tanh layer + * + * *Implementation of ailayer.forward.* + * + * It uses the result tensor of the previous layer as input and writes the result of the forward pass + * to the result tensor (ailayer.result) of the given layer. + * + * Calculation of the forward pass result: + * @f[ + * x_{out} \leftarrow tanh(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ x_{out} \f$: Result of the forward pass of this layer\n\n + * + * Used math functions: + * * ailayer_tanh.tanh + * + * @param *self Layer to calculate the forward path for. + */ +void ailayer_tanh_forward(ailayer_t *self); + +/** @brief Calculate the backward pass for the given Tanh layer + * + * *Implementation of ailayer.backward.* + * + * It uses the deltas tensor of the next layer as input and writes the result of the backward pass + * to the deltas tensor (ailayer.deltas) of the given layer. + * + * Calculation of the errors for the previous layer: + * @f[ + * \delta_{in} \leftarrow \delta_{out} \circ tanh'(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ \delta_{in} \f$: Result of the backward pass of this layer\n + * \f$ \delta_{out} \f$: Result of the backward pass of the next layer\n\n + * + * Used math functions: + * * ailayer_tanh.tanh + * * ailayer_tanh.d_tanh + * * ailayer_tanh.multiply + * + * @param *self Layer to calculate the backward path for. + */ +void ailayer_tanh_backward(ailayer_t *self); + +/** @brief Calculate the shape of the result tensor + * + * *Implementation of ailayer.calc_result_shape.* + * + * As the result tensor shape is shared with the result tensor shape of the previous layer (no change in shape is needed), + * this function returns without doing anything. + * + * @param *self Layer to calculate the resulting shape for. + */ +void ailayer_tanh_calc_result_shape(ailayer_t *self); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the layer specification + * + * @param *self The layer to print the specification for + * @param *print Pointer to the print function to use + */ +void ailayer_tanh_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // TANH_LAYER diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_template.c b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_template.c new file mode 100644 index 0000000..2c39b9b --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_template.c @@ -0,0 +1,211 @@ +/** + * \file basic/base/ailayer/ailayer_template.c + * \version 2.0alpha + * \date 20.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/ailayer/ailayer_template.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_layertype_t ailayer_template_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "Template", + .print_specs = ailayer_template_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_layertype_t *ailayer_template_type = &ailayer_template_type_s; + +ailayer_t *ailayer_template(ailayer_template_t *layer, ailayer_t *input_layer) +{ + // Connect the layer with its input + layer->base.input_layer = input_layer; + input_layer->output_layer = &(layer->base); + + // Init basic layer variables + layer->base.layer_configuration = layer; + layer->base.result.dtype = layer->dtype; + layer->base.result.shape = layer->result_shape; + layer->base.result.dim = 2; + + layer->base.deltas.dtype = layer->dtype; + layer->base.deltas.shape = input_layer->result.shape; + layer->base.deltas.dim = 2; + + // Set the function pointers + layer->base.forward = ailayer_template_forward; + layer->base.backward = ailayer_template_backward; + + layer->base.calc_result_shape = ailayer_template_calc_result_shape; + layer->base.sizeof_paramem = ailayer_template_sizeof_paramem; + layer->base.set_paramem = ailayer_template_set_paramem; + layer->base.sizeof_paramem = ailayer_template_sizeof_trainmem; + layer->base.set_trainmem = ailayer_template_set_trainmem; + + return &layer->base; +} + +/* +* Layer naming: +* ------- +* x_in -> | L | -> x_out +* | A | +* | Y | +* | E | +* delta_in <- | R | <- delta_out +* ------- +*/ + + +void ailayer_template_forward(ailayer_t *self) +{ + ailayer_template_t *layer = (ailayer_template_t *)(self->layer_configuration); + aitensor_t *x_in = &(self->input_layer->result); + aitensor_t *x_out = &(self->result); + + // Example forward: x_out = x_in + params + layer->tensor_add(x_in, layer->params, x_out); + + return; +} + +void ailayer_template_backward(ailayer_t *self) +{ + ailayer_template_t *layer = (ailayer_template_t *)(self->layer_configuration); + aitensor_t *x_in = &(self->input_layer->result); +// aitensor_t *x_out = &(self->result); + aitensor_t *delta_in = &(self->deltas); + aitensor_t *delta_out = &(self->output_layer->deltas); + + // Temporary result for calculation can be created here. Document the needed amount of temp memory! + // Remove code if unused! + float temp_result_data[delta_out->shape[0] * delta_out->shape[1]]; + uint16_t temp_result_shape[] = {delta_out->shape[0], delta_out->shape[1]}; + aitensor_t temp_result = { + .dim = 2, + .shape = temp_result_shape, + .data = temp_result_data, + .dtype = x_in->dtype + }; + + // 1) Calculate the gradients and add to the gradients tensor + layer->copy_tensor(delta_out, &temp_result); // (Unnecessary, just show the use of temp_result) + layer->tensor_add(layer->d_params, &temp_result, layer->d_params); + + // 2) Calculate delta for next layer. Do not before calculating gradients!!! May override x_in!!! + layer->copy_tensor(&temp_result, delta_in); + + return; +} + +void ailayer_template_calc_result_shape(ailayer_t *self) +{ +// ailayer_template_t *layer = (ailayer_template_t *)(self->layer_configuration); + aitensor_t *x_in = &(self->input_layer->result); + aitensor_t *x_out = &(self->result); + + x_out->shape[0] = x_in->shape[0]; + x_out->shape[1] = x_in->shape[1]; + + return; +} + +uint32_t ailayer_template_sizeof_paramem(const ailayer_t *self) +{ + ailayer_template_t *layer = (ailayer_template_t *)(self->layer_configuration); +// const aitensor_t *x_in = &(self->input_layer->result); + const aitensor_t *x_out = &(self->result); + + uint32_t memory = 0; + + // Memory amount for params. Attention: result shape is calculated but params tensor is not available yet. + memory += sizeof(aitensor_t); // struct + memory += 2 * sizeof(uint16_t); // shape array + memory += x_out->shape[0] * x_out->shape[1] * aimath_sizeof_dtype(layer->dtype); // data + + return memory; +} + +void ailayer_template_set_paramem(ailayer_t *self, void *memory_ptr) +{ + ailayer_template_t *layer = (ailayer_template_t *)(self->layer_configuration); + aitensor_t *x_in = &(self->input_layer->result); +// aitensor_t *x_out = &(self->result); + + uint32_t address_counter = 0; + + // Params memory distribution + // tensor struct: + layer->params = memory_ptr + address_counter; + address_counter += sizeof(aitensor_t); + layer->params->dim = 2; + layer->params->dtype = layer->dtype; + // shape array: + layer->params->shape = memory_ptr + address_counter; + address_counter += 2 * sizeof(uint16_t); + layer->params->shape[0] = x_in->shape[0]; + layer->params->shape[1] = x_in->shape[1]; + // data: + layer->params->data = memory_ptr + address_counter; + address_counter += aimath_sizeof_tensor_data(layer->params); + + return; +} + +uint32_t ailayer_template_sizeof_trainmem(const ailayer_t *self) +{ + ailayer_template_t *layer = (ailayer_template_t *)(self->layer_configuration); + + uint32_t memory = 0; + + // Sum up the memory sizes for every parameter tensor + memory += aimath_sizeof_tensor(layer->params); // Size of tensor struct and data but not the shape array, because shape array is shared with params + + return memory; +} + +void ailayer_template_set_trainmem(ailayer_t *self, void *memory_ptr) +{ + ailayer_template_t *layer = (ailayer_template_t *)(self->layer_configuration); + + uint32_t address_counter = 0; + + // Params gradient memory distribution + // tensor struct: + layer->d_params = memory_ptr + address_counter; + address_counter += sizeof(aitensor_t); + layer->d_params->dim = 2; + layer->d_params->shape = layer->params->shape; // shared shape + // data: + layer->d_params->data = memory_ptr + address_counter; + address_counter += aimath_sizeof_tensor_data(layer->d_params); + + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void ailayer_template_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)) +{ + return; +} +#endif diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_template.h b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_template.h new file mode 100644 index 0000000..c5f28d4 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailayer/ailayer_template.h @@ -0,0 +1,131 @@ +/** + * \file basic/base/ailayer/ailayer_template.h + * \internal + * \date 20.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief [DEPRECATED] This file acts as a reference for new layer structures. + * Some of the shown functions are mandatory, some are optional. + * \details + */ + +#ifndef TEMPLATE_LAYER +#define TEMPLATE_LAYER + +#include "core/aifes_core.h" + +typedef struct ailayer_template ailayer_template_t; + +/** @brief Template for general AIfES layers +* +*/ +struct ailayer_template { + ailayer_t base; /**< Inherited field members from general ailayer struct. */ // The base or super "class" of the layer. + const aimath_dtype_t *dtype; /**< Datatype of the inference result values. */ + + // Configurations of the layer, that can be configured by the user when creating the layer + uint32_t example_configuration; + + // Tensor pointer for trainable parameters [DEPRECATED] + aitensor_t *params; + aitensor_t *d_params; // Gradients + + // If the shape of the result differs from the input shape, this array can be used for the result. + // If not, the result shape array of the input tensor can be used for the result tensor too. + uint16_t result_shape[2]; /**< Inference result tensor shape (n x m matrix). */ + + aitensor_t *trainable_params[2]; /**< Pointer to the weights and biases (which are the trainable parameters). */ + aitensor_t *gradients[2]; /**< Gradients structure for the back propagation algorithm. */ + void *optimem[2]; /**< Memory field used by the trainings optimizer. */ + + // Make the math functions accessable for expert developers + void (*copy_tensor)(const aitensor_t *from, aitensor_t *to); /**< Function pointer to the used tensor copy function. */ + void (*tensor_add)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); /**< Function pointer to the used tensor addition. */ +}; + +extern const aicore_layertype_t *ailayer_template_type; +/** @brief Initialize the given layer struct - __mandatory__ + * + * Initializes and sets all parameters needed by the layer structure. + * + * @param *layer The layer to initialize. + * @param *input_layer The layer that provides the inputs to the initialized layer. + * @return Pointer to the (successfully) initialized layer structure. + */ + +ailayer_t *ailayer_template(ailayer_template_t *layer, ailayer_t *input_layer); + +/** @brief Calculate the forward path for given layer - __mandatory__ + * + * @param *self Layer to calculate the forward path for. + */ +void ailayer_template_forward(ailayer_t *self); + +/** @brief Calculate the backward (training) path for the given layer - __depends on usability in training__ + * + * @param *self Layer to calculate the backward path for. + */ +void ailayer_template_backward(ailayer_t *self); + +/** @brief Calculate the shape of the result tensor + * + * @param *self Layer to calculate the resulting shape for. + */ +void ailayer_template_calc_result_shape(ailayer_t *self); + +/** @brief Calculate and return the parameter memory size + * needed for this layer - __mandatory for hidden layers__ + * + * @param *self The layer to calculate the parameter memory size for + * @return Calculated parameter memory size in bytes. + */ +uint32_t ailayer_template_sizeof_paramem(const ailayer_t *self); + +/** @brief Distribute provided memory to the parameter pointers - __mandatory for hidden layers__ + * + * @param *self The layer to set the memory fields for. + * @param *memory_ptr Pointer to the buffer provided for the layer parameters. + */ +void ailayer_template_set_paramem(ailayer_t *self, void *memory_ptr); + +/** @brief Calculate and return the necessary memory size + * needed by this layer for training - __mandatory for hidden layers__ + * + * @param *self The layer to calculate the gradient memory size for. + * @return Calculated gradient memory size in bytes. + */ +uint32_t ailayer_template_sizeof_trainmem(const ailayer_t *self); + +/** @brief Distribute provided memory to the training gradients pointers - __mandatory for hidden layers__ + * + * @param *self The layer to set the memory fields for. + * @param *memory_ptr Pointer to the buffer provided for the layer gradients. + */ +void ailayer_template_set_trainmem(ailayer_t *self, void *memory_ptr); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the layer specification + * + * @param *self The layer to print the specification for + * @param *print Pointer to the print function to use + */ +void ailayer_template_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)); +#endif + +#endif // TEMPLATE_LAYER diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailoss/ailoss_crossentropy.c b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailoss/ailoss_crossentropy.c new file mode 100644 index 0000000..1591d70 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailoss/ailoss_crossentropy.c @@ -0,0 +1,110 @@ +/** + * \file basic/base/ailoss/ailoss_crossentropy.c + * \version 2.0alpha + * \date 12.01.2021 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/ailoss/ailoss_crossentropy.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_losstype_t ailoss_crossentropy_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "Cross-entropy", + .print_specs = ailoss_crossentropy_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_losstype_t *ailoss_crossentropy_type = &ailoss_crossentropy_type_s; + +ailoss_t *ailoss_crossentropy(ailoss_crossentropy_t *loss, ailayer_t *input_layer) +{ + loss->base.loss_type = ailoss_crossentropy_type; + + loss->base.connection_layer.input_layer = input_layer; + input_layer->output_layer = &(loss->base.connection_layer); + + loss->base.loss_configuration = loss; + + loss->base.connection_layer.deltas.dtype = loss->dtype; + + loss->base.calc_delta = ailoss_crossentropy_calc_delta; + loss->base.calc_loss = ailoss_crossentropy_calc_loss; + + // Check for valid input and override the backward function (not needed) + if(input_layer->layer_type == ailayer_softmax_type || input_layer->layer_type == ailayer_sigmoid_type){ + input_layer->backward = ailoss_crossentropy_dummy_backward; + } + else{ + #ifdef AIDEBUG_PRINT_ERROR_MESSAGES + printf("\n!!! ERROR !!! (ailoss_crossentropy): No valid input layer. Use either Sigmoid or Softmax as input.\n"); + #endif + return 0; + } + + return &loss->base; +} + +void ailoss_crossentropy_calc_delta(ailoss_t *self, const aitensor_t *target_data) +{ + ailoss_crossentropy_t *loss = (ailoss_crossentropy_t *)(self->loss_configuration); + aitensor_t *predicted_data = &(self->connection_layer.input_layer->result); + aitensor_t *deltas = &(self->connection_layer.input_layer->deltas); + + deltas->shape = predicted_data->shape; // ToDo: Maybe remove or put it in constructor + + // Calc dC/dz directly instead of dC/da_L and then da_L/dz in the output layer + // dC/dz = (a_L - y) ( * 1/batch_size) + loss->tensor_sub(predicted_data, target_data, deltas); + + // ToDo: Scale by batch size. (Divide / Shift) + + return; +} + +void ailoss_crossentropy_calc_loss(ailoss_t *self, const aitensor_t *target_data, void *result) +{ + ailoss_crossentropy_t *loss = (ailoss_crossentropy_t *)(self->loss_configuration); + + aitensor_t *predicted_data = &(self->connection_layer.input_layer->result); + + //C = - y * log(a_L) ( * 1/batch_size) + loss->crossentropy(predicted_data, target_data, result); + + // ToDo: Scale by batch size. (Divide / Shift) + + return; +} + +void ailoss_crossentropy_dummy_backward(ailayer_t *self) +{ + return; +} + + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void ailoss_crossentropy_print_specs(const ailoss_t *self, int (*print)(const char *format, ...)) +{ + return; +} +#endif + diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailoss/ailoss_crossentropy.h b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailoss/ailoss_crossentropy.h new file mode 100644 index 0000000..f8cb30e --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailoss/ailoss_crossentropy.h @@ -0,0 +1,205 @@ +/** + * \file basic/base/ailoss/ailoss_crossentropy.h + * \internal + * \date 12.01.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link ailoss loss \endlink implementation of the Cross-Entropy loss + * + * This is an "abstract" data-type independent implementation. To use the loss, use one of the provided + * implementations for a specific hardware and data-type (for example from ailoss_crossentropy_default.h) or set + * the required math functions on your own. + * + * The Cross-Entropy loss ist best suitable for classification tasks + * and works only with \link ailayer_sigmoid.h Sigmoid \endlink or \link ailayer_softmax.h Softmax \endlink output layers. + * + * For **binary classification** (binary targets 0 or 1) with **Sigmoid output layer**, the loss / cost is calculated as + * @f[ + * L(y, \hat{y}) = - \sum_{i=0}^{N} (y_i \log(\hat{y}_i) + (1-y) \log(1-\hat{y}_i)) + * @f] + * with the predicted values \f$ \hat{y}_i \f$ and the target values \f$ y_i \f$. + * \f$ N \f$ is the number of elements of the \f$ y \f$ tensor. + * + * For **categorigal classification** (one-hot encoded targets) with **Softmax output layer**, the loss / cost is calculated as + * @f[ + * L(y, \hat{y}) = - \sum_{i=0}^{N} y_{i} \log(\hat{y}_{i}) + * @f] + * with the predicted values \f$ \hat{y}_i \f$ and the target values \f$ y_i \f$. + * \f$ N \f$ is the number of elements of the \f$ y \f$ tensor. + * + * To get the "mean" normalization, you have to modify the learning rate to \f$ lr = \frac {1}{o \cdot n} \cdot lr \f$ + * with the number of outputs \f$ o \f$ and the batch size \f$ n \f$. + * + * The loss can be calculated with ailoss_crossentropy_calc_loss(). For training the deltas /errors on the target data are + * calculated with ailoss_crossentropy_calc_delta() and written to the deltas tensor of the connection layer. + */ + +#ifndef CROSSENTROPY_LOSS +#define CROSSENTROPY_LOSS + +#include "core/aifes_core.h" +#include "basic/base/ailayer/ailayer_softmax.h" +#include "basic/base/ailayer/ailayer_sigmoid.h" + +typedef struct ailoss_crossentropy ailoss_crossentropy_t; /**< New data type name for code reduction. */ + +/** @brief General \link ailoss_crossentropy.h Cross-Entropy loss \endlink struct +* +*/ +struct ailoss_crossentropy { + ailoss_t base; /**< Inherited field members from general ailoss struct. */ + const aimath_dtype_t *dtype; /**< Main data type of the loss. */ + + /** @name Math functions + * @brief Required data type specific math functions + */ + ///@{ + + /** @brief Required math function: Element wise tensor subtraction + * + * Requires a math function that subtracts two tensors element wise:\n + * @f[ + * result = a - b + * @f] + */ + void (*tensor_sub)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + /** @brief Required math function: Cross-Entropy between two tensors + * + * Requires a math function that calculates the Cross-Entropy between two tensors.\n + * For a Sigmoid output layer it should be the binary Cross-Entropy: + * @f[ + * result = -\sum (target \cdot \log(predicted) + (1 - target) \log(1 - predicted)) + * @f] + * + * For a Softmax output layer it should be the categorical Cross-Entropy: + * @f[ + * result = -\sum target \cdot \log(predicted) + * @f] + * + * @param result A Scalar of the defined data type + */ + void (*crossentropy)(const aitensor_t *predicted_data, const aitensor_t *target_data, void *result); + + ///@} +}; + +/** @brief Cross-Entropy loss type + * + * Defines the type of the loss (for example for type checks and debug prints). + * See aicore_losstype for more information about the loss type. + */ +extern const aicore_losstype_t *ailoss_crossentropy_type; + +/** @brief Initialize and connect the given Cross-Entropy loss + * + * This function represents the "constructor" of the abstract Cross-Entropy loss. It initializes the loss structure + * and connects it to the output layer of the AIfES model.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example ailoss_crossentropy_f32_default()). + * + * @param *loss The loss to initialize. + * @param *input_layer The output layer of the model that provides the inputs to the loss. + * @return Pointer to the (successfully) initialized loss structure. + */ +ailoss_t *ailoss_crossentropy(ailoss_crossentropy_t *loss, ailayer_t *input_layer); + +/** @brief Calculate the combined derivative of the given Cross-Entropy loss and the output layer for error backpropagation + * + * *Implementation of ailoss.calc_delta.* + * + * It uses the result tensor of the output layer and the target data as input and writes the result + * to the deltas tensor (ailayer.deltas) of the output layer of the model. + * + * By combining the Cross-Entropy loss with a Sigmoid or Softmax output layer, the combined deltas can be + * calculated very efficiently. The backward function of the Sigmoid / Softmax layer is not used anymore. + * + * Calculation of the deltas: + * @f[ + * \delta_{in} \leftarrow p - y + * @f] + * + * \f$ \delta_{in} \f$: Result of the delta calculation of this loss (written to ailayer.deltas of the output layer of the model)\n + * \f$ p \f$: Result of the forward pass of the output layer of the model (predicted values)\n + * \f$ y \f$: Target data / True values / Labels\n\n + * + * Used math functions: + * * ailoss_crossentropy.tensor_sub + * + * @param *self Loss to calculate the deltas for + * @param *target_data Target data / True values / Labels + */ +void ailoss_crossentropy_calc_delta(ailoss_t *self, const aitensor_t *target_data); + +/** @brief Calculate the Cross-Entropy loss on the given target data + * + * *Implementation of ailoss.calc_loss.* + * + * It uses the result tensor of the output layer and the target data as input and writes the result + * to the given result scalar. + * + * Calculation of the loss with Sigmoid output layer: + * @f[ + * result \leftarrow - \sum_i (y_i \log(p_i) + (1 - y_i) \log(1 - p_i)) + * @f] + * + * Calculation of the loss with Softmax output layer: + * @f[ + * result \leftarrow - \sum_i y_i \log(p_i) + * @f] + * + * + * \f$ result \f$: Result of the loss calculation\n + * \f$ p \f$: Result of the forward pass of the output layer of the model (predicted values)\n + * \f$ y \f$: Target data / True values / Labels\n\n + * + * Used math functions: + * * ailoss_crossentropy.tensor_sub + * * ailoss_crossentropy.crossentropy + * + * @param *self Loss to calculate the deltas for + * @param *target_data Target data / True values / Labels + * @param *result Result scalar (the data type is specified by the data type specific implementations) + */ +void ailoss_crossentropy_calc_loss(ailoss_t *self, const aitensor_t *target_data, void *result); + +/** @brief Dummy backward-function for the output layer of the model + * + * *Implementation of ailayer.backward.* +* +* The ailoss_crossentropy_calc_delta() function calculates the combined delta for a Sigmoid or Softmax layer +* with Cross-Entropy loss. Therefor the backward function of the Sigmoid / Softmax layer is not needed anymore +* and gets exchanged by this dummy function. +* +* The dummy function does nothing. +* +* @param *self The output layer of the model +*/ +void ailoss_crossentropy_dummy_backward(ailayer_t *self); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the loss specification + * + * @param *self The loss to print the specification for + * @param *print Pointer to the print function to use + */ +void ailoss_crossentropy_print_specs(const ailoss_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // CROSSENTROPY_LOSS diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailoss/ailoss_mse.c b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailoss/ailoss_mse.c new file mode 100644 index 0000000..f42d37c --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailoss/ailoss_mse.c @@ -0,0 +1,99 @@ +/** + * \file basic/base/ailoss/ailoss_mse.c + * \version 2.0alpha + * \date 20.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/ailoss/ailoss_mse.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_losstype_t ailoss_mse_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "Mean Squared Error", + .print_specs = ailoss_mse_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_losstype_t *ailoss_mse_type = &ailoss_mse_type_s; + +ailoss_t *ailoss_mse(ailoss_mse_t *loss, ailayer_t *input_layer) +{ + loss->base.loss_type = ailoss_mse_type; + + loss->base.connection_layer.input_layer = input_layer; + input_layer->output_layer = &(loss->base.connection_layer); + + loss->base.loss_configuration = loss; + + loss->base.connection_layer.deltas.dtype = loss->dtype; + loss->base.connection_layer.deltas.dim = 2; + loss->base.connection_layer.deltas.shape = input_layer->result.shape; + loss->base.connection_layer.get_result_bound = 0; + + loss->base.calc_delta = ailoss_mse_calc_delta; + loss->base.calc_loss = ailoss_mse_calc_loss; + + return &loss->base; +} + +void ailoss_mse_calc_delta(ailoss_t *self, const aitensor_t *target_data) +{ + ailoss_mse_t *loss = (ailoss_mse_t *)(self->loss_configuration); + aitensor_t *predicted_data = &(self->connection_layer.input_layer->result); + + loss->tensor_sub(predicted_data, target_data, &(self->connection_layer.deltas)); + + // ToDo: Scale by batch size. (Divide / Shift) + + return; +} + +void ailoss_mse_calc_loss(ailoss_t *self, const aitensor_t *target_data, void *result) +{ + ailoss_mse_t *loss = (ailoss_mse_t *)(self->loss_configuration); + + aitensor_t *predicted_data = &(self->connection_layer.input_layer->result); + + uint8_t temp_tensor_data[aimath_sizeof_tensor_data(target_data)]; + aitensor_t temp_tensor = { + .dim = target_data->dim, + .shape = target_data->shape, + .data = temp_tensor_data, + .dtype = target_data->dtype, + .tensor_params = self->connection_layer.deltas.tensor_params + }; + + loss->tensor_sub(predicted_data, target_data, &temp_tensor); + loss->norm_squared(&temp_tensor, result); + + // ToDo: Scale by batch size. (Divide / Shift) + + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void ailoss_mse_print_specs(const ailoss_t *self, int (*print)(const char *format, ...)) +{ + return; +} +#endif diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailoss/ailoss_mse.h b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailoss/ailoss_mse.h new file mode 100644 index 0000000..bae272e --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/ailoss/ailoss_mse.h @@ -0,0 +1,162 @@ +/** + * \file basic/base/ailoss/ailoss_mse.h + * \internal + * \date 20.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link ailoss loss \endlink implementation of the Mean Squared Error (MSE) loss + * + * This is an "abstract" data-type independent implementation. To use the loss, use one of the provided + * implementations for a specific hardware and data-type (for example from ailoss_mse_default.h) or set + * the required math functions on your own. + * + * The Mean Squared Error (MSE) loss ist best suitable for regression tasks. The loss / cost is calculated as + * @f[ + * L(y, \hat{y}) = \sum_{i=0}^{N} (y_i - \hat{y}_i)^2 + * @f] + * with the predicted values \f$ \hat{y}_i \f$ and the target values \f$ y_i \f$. + * \f$ N \f$ is the number of elements of the \f$ y \f$ tensor. + * + * To get the "mean" normalization, you have to modify the learning rate to \f$ lr = \frac {1}{o \cdot n} \cdot lr \f$ + * with the number of outputs \f$ o \f$ and the batch size \f$ n \f$. + * + * The loss can be calculated with ailoss_mse_calc_loss(). For training the deltas /errors on the target data are + * calculated with ailoss_mse_calc_delta() and written to the deltas tensor of the connection layer. + */ + +#ifndef MSE_LOSS +#define MSE_LOSS + +#include "core/aifes_core.h" + +typedef struct ailoss_mse ailoss_mse_t; /**< New data type name for code reduction. */ + +/** @brief General \link ailoss_mse.h Mean Squared Error (MSE) loss \endlink struct + * + */ +struct ailoss_mse { + ailoss_t base; /**< Inherited field members from general ailoss struct. */ + const aimath_dtype_t *dtype; /**< Main data type of the loss. */ + + /** @name Math fuctions + * @brief Required data type specific math functions + */ + ///@{ + + /** @brief Required math function: Element wise tensor subtraction + * + * Requires a math function that subtracts two tensors element wise:\n + * @f[ + * result = a - b + * @f] + */ + void (*tensor_sub)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + /** @brief Required math function: Squared sum of tensor elements + * + * Requires a math function that calculates the squared sum of all elements of a given tensor:\n + * @f[ + * result = \sum_i x_{i}^2 + * @f] + */ + void (*norm_squared)(const aitensor_t *x, void *result); + + ///@} +}; + +/** @brief Mean Squared Error loss type + * + * Defines the type of the loss (for example for type checks and debug prints). + * See aicore_losstype for more information about the loss type. + */ +extern const aicore_losstype_t *ailoss_mse_type; + +/** @brief Initialize and connect the given MSE loss + * + * This function represents the "constructor" of the abstract MSE loss. It initializes the loss structure + * and connects it to the output layer of the AIfES model.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example ailoss_mse_f32_default()). + * + * @param *loss The loss to initialize. + * @param *input_layer The output layer of the model that provides the inputs to the loss. + * @return Pointer to the (successfully) initialized loss structure. + */ +ailoss_t *ailoss_mse(ailoss_mse_t *loss, ailayer_t *input_layer); + +/** @brief Calculate the derivative of the given MSE loss for error backpropagation + * + * *Implementation of ailoss.calc_delta.* + * + * It uses the result tensor of the output layer and the target data as input and writes the result + * to the deltas tensor (ailayer.deltas) of the connection layer (ailoss.connection_layer). + * + * Calculation of the deltas: + * @f[ + * \delta_{in} \leftarrow p - y + * @f] + * + * \f$ \delta_{in} \f$: Result of the delta calculation of this loss (written to ailayer.deltas of the ailoss.connection_layer)\n + * \f$ p \f$: Result of the forward pass of the output layer of the model (predicted values)\n + * \f$ y \f$: Target data / True values / Labels\n\n + * + * Used math functions: + * * ailoss_mse.tensor_sub + * + * @param *self Loss to calculate the deltas for + * @param *target_data Target data / True values / Labels + */ +void ailoss_mse_calc_delta(ailoss_t *self, const aitensor_t *target_data); + +/** @brief Calculate the MSE loss on the given target data + * + * *Implementation of ailoss.calc_loss.* + * + * It uses the result tensor of the output layer and the target data as input and writes the result + * to the given result scalar. + * + * Calculation of the loss: + * @f[ + * result \leftarrow \sum_i (y_i - p_i)^2 + * @f] + * + * \f$ result \f$: Result of the loss calculation\n + * \f$ p \f$: Result of the forward pass of the output layer of the model (predicted values)\n + * \f$ y \f$: Target data / True values / Labels\n\n + * + * Used math functions: + * * ailoss_mse.tensor_sub + * * ailoss_mse.norm_squared + * + * @param *self Loss to calculate the deltas for + * @param *target_data Target data / True values / Labels + * @param *result Result scalar (the data type is specified by the data type specific implementations) + */ +void ailoss_mse_calc_loss(ailoss_t *self, const aitensor_t *target_data, void *result); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the loss specification + * + * @param *self The loss to print the specification for + * @param *print Pointer to the print function to use + */ +void ailoss_mse_print_specs(const ailoss_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // MSE_LOSS diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aimath/aimath_basic.c b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aimath/aimath_basic.c new file mode 100644 index 0000000..e927504 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aimath/aimath_basic.c @@ -0,0 +1,78 @@ +/** + * \file basic/base/aimath/aimath_basic.c + * \version 2.0alpha + * \date 28.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Basic math operations for aitensor_t + * \details Basic functions for aitensor_t handling, e.g. calculation of size, printing etc. + */ + +#include "basic/base/aimath/aimath_basic.h" + +void print_aitensor(const aitensor_t *tensor) +{ + tensor->dtype->print_aitensor(tensor); + return; +} + +void print_aiscalar(const void *scalar, const aimath_dtype_t *dtype) +{ + dtype->print_aiscalar(scalar, printf); + return; +} + +void aimath_transpose_vector(aitensor_t *vector) +{ + uint16_t temp; + temp = vector->shape[0]; + vector->shape[0] = vector->shape[1]; + vector->shape[1] = temp; + return; +} + +uint32_t aimath_tensor_elements(const aitensor_t *tensor) +{ + uint32_t elems = 1; + uint8_t i; + + for(i = 0; i < tensor->dim; i++) + { + elems *= tensor->shape[i]; + } + return elems; +} + +uint32_t aimath_sizeof_dtype(const aimath_dtype_t *dtype) +{ + return dtype->size; +} + +uint32_t aimath_sizeof_tensor_data(const aitensor_t *tensor) +{ + return aimath_sizeof_dtype(tensor->dtype) * aimath_tensor_elements(tensor); +} + +uint32_t aimath_sizeof_tensor_params(const aitensor_t *tensor) +{ + return tensor->dtype->tensor_params_size; +} + +uint32_t aimath_sizeof_tensor(const aitensor_t *tensor) +{ + return sizeof(aitensor_t) + aimath_sizeof_tensor_data(tensor) + aimath_sizeof_tensor_params(tensor); +} diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aimath/aimath_basic.h b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aimath/aimath_basic.h new file mode 100644 index 0000000..178d702 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aimath/aimath_basic.h @@ -0,0 +1,123 @@ +/** + * \file basic/base/aimath/aimath_basic.h + * \internal + * \date 28.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Basic data-type independent math operations + * + */ + +#ifndef AIMATH_BASIC +#define AIMATH_BASIC + +#include "core/aifes_math.h" + +/** @brief Printing a tensor to console + * + * Calls the corresponding print function of the used aimath_dtype. + * + * Example: + * \code{.c} + * uint16_t tensor_shape[2] = {2, 3}; + * float tensor_data[2*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f}; + * aitensor_t tensor = AITENSOR_2D_F32(tensor_shape, tensor_data); + * + * print_aitensor(&tensor); + * \endcode + * + * @param *tensor The tensor to print. + */ +void print_aitensor(const aitensor_t *tensor); + +/** @brief Printing a scalar to console + * + * Calls the corresponding print function of the used aimath_dtype. + * + * Example: + * \code{.c} + * aiscalar_f32_t scalar = 42.0f; + * + * print_aiscalar(&scalar, aif32); + * \endcode + * + * @param *scalar The scalar to print. + * @param *dtype The data type of the scalar + */ +void print_aiscalar(const void *scalar, const aimath_dtype_t *dtype); + +/** @brief Transposes a vector + * + * @f[ + * vector \leftarrow vector^T + * @f] + * + * Make sure that the given tensor is a vector (with shape [1 x N] or [N x 1]) + * + * @param *vector The vector to transpose (2D tensor of shape [1 x N] or [N x 1]) + */ +void aimath_transpose_vector(aitensor_t *vector); + +/** @brief Calculates the number of elements in a tensor + * + * @param *tensor The tensor to count the elements of + * @return Number of elements in tensor + */ +uint32_t aimath_tensor_elements(const aitensor_t *tensor); + +/** @brief Returns the size of one value in the given data type in bytes + * + * @param *dtype The data type to get the size of + * @return Size of one value in the given data type in bytes + */ +uint32_t aimath_sizeof_dtype(const aimath_dtype_t *dtype); + +/** @brief Returns size of the data array of the tensor in bytes + * + * The size is calculated by: + * + * \code aimath_tensor_elements(tensor) * aimath_sizeof_dtype(tensor.dtype) \endcode + * + * @param *tensor The tensor to get the data size of + * @return Size of tensor data in bytes + */ +uint32_t aimath_sizeof_tensor_data(const aitensor_t *tensor); + +/** @brief Returns size of the aitensor.tensor_params for the data type of the tensor + * + * @param *tensor The tensor to get the parameters size of + * @return Size of tensor parameters in bytes + */ +uint32_t aimath_sizeof_tensor_params(const aitensor_t *tensor); + +/** @brief Returns size of tensor in bytes + * + * The size is calculated by: + * + * \code sizeof(aitensor_t) + aimath_sizeof_tensor_data(tensor) + aimath_sizeof_tensor_params(tensor) \endcode + * + * @param *tensor The tensor to get the size of + * @return Size of tensor in bytes + */ +uint32_t aimath_sizeof_tensor(const aitensor_t *tensor); + +//void aimath_create_tensor_2d(uint16_t shape_0, uint16_t shape_1, aimath_dtype_t dtype, void *memory_ptr); + +#endif // AIMATH_BASIC diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aimath/aimath_f32.c b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aimath/aimath_f32.c new file mode 100644 index 0000000..0eaa402 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aimath/aimath_f32.c @@ -0,0 +1,91 @@ +/** + * \file basic/base/aimath/aimath_f32.c + * \version 2.0alpha + * \date 28.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Basic functions for f32 datatypes + */ + +#include "basic/base/aimath/aimath_f32.h" + +const aimath_dtype_t aif32_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "F32", +#else + .name = 0, +#endif + .size = 4, + .tensor_params_size = 0, + .print_aitensor = aimath_f32_print_aitensor, + .print_aiscalar = aimath_f32_print_aiscalar +}; + +const aimath_dtype_t *aif32 = &aif32_s; + + +void aimath_f32_print_aitensor(const aitensor_t *tensor) +{ + uint16_t i, j, k, n; + printf("F32 [\n"); + if(tensor->dim == 1) + { + for(j = 0; j < tensor->shape[0]; j++) + { + printf("%10.5f\t", ((float *) tensor->data)[j]); + } + } + else if(tensor->dim == 2) + { + for(i = 0; i < tensor->shape[0]; i++) + { + for(j = 0; j < tensor->shape[1]; j++) + { + printf("%10.5f\t", ((float *) tensor->data)[i*tensor->shape[1] + j]); + } + printf("\n"); + } + } + else if(tensor->dim == 4) + { + float (*tensor_data)[tensor->shape[0]][tensor->shape[1]][tensor->shape[2]][tensor->shape[3]] = tensor->data; + + for(n = 0; n < tensor->shape[0]; n++) + { + for(k = 0; k < tensor->shape[1]; k++) + { + for(i = 0; i < tensor->shape[2]; i++) + { + for(j = 0; j < tensor->shape[3]; j++) + { + printf("%10.5f\t", (*tensor_data)[n][k][i][j]); + } + printf("\n"); + } + printf("\n"); + } + printf("\n"); + } + } + printf("]\n"); + return; +} + +void aimath_f32_print_aiscalar(const void *scalar, int (*print)(const char *format, ...)) +{ + print("%f (F32)", *((float *) scalar)); +} diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aimath/aimath_f32.h b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aimath/aimath_f32.h new file mode 100644 index 0000000..e067fe8 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aimath/aimath_f32.h @@ -0,0 +1,134 @@ +/** + * \file basic/base/aimath/aimath_f32.h + * \internal + * \date 28.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Definition of the F32 (aif32) data-type + * + * The F32 (aif32) data-type stores data as 32 bit single precision floating point values. + * It does not require any additional parameter to define the values. + * + * **Example: Create a F32 tensor**\n + * The tensor + * @f[ + * \left( \begin{array}{rrr} 0 & 1 & 2 \\ 3 & 4 & 5 \end{array}\right) + * @f] + * can be created with + * \code{.c} + * float example_data[] = {0.0f, 1.0f, 2.0f, + * 3.0f, 4.0f, 5.0f}; + * uint16_t example_shape[] = {2, 3}; + * aitensor_t example_tensor = { + * .dtype = aif32, + * .dim = 2, + * .shape = example_shape, + * .data = example_data + * }; + * \endcode + * + * **Example: Create a F32 scalar**\n + * Either create it as a normal float value + * \code{.c} + * float scalar = 42.0f; + * \endcode + * or with the alias + * \code{.c} + * aiscalar_f32_t scalar = 42.0f; + * \endcode + * + * **Example: Print a F32 tensor to the console** + * \code{.c} + * print_aitensor(&example_tensor); + * \endcode + * + * **Example: Print a F32 scalar to the console** + * \code{.c} + * print_aiscalar(&scalar, aif32); + * \endcode + */ + +#ifndef AIMATH_F32 +#define AIMATH_F32 + +#include "core/aifes_math.h" +#include "basic/base/aimath/aimath_basic.h" + +/** @brief Initialize a 2 dimensional F32 tensor + * + * @param shape A uint16_t array of length 2 for the shape + * @param data A float array for the tensor data + */ +#define AITENSOR_2D_F32(shape, data) {aif32, 2, shape, 0, data} + +/** @brief Scalar for F32 (aif32) data-type + * + * This is only an alias for a float value for consistency.\n + * You can create a F32 scalar either as a normal float value + * \code{.c} + * float scalar = 42.0f; + * \endcode + * or with the alias + * \code{.c} + * aiscalar_f32_t scalar = 42.0f; + * \endcode + * + * You can print the scalar to the console with + * \code{.c} + * print_aiscalar(&scalar, aif32); + * \endcode + * or by using + * \code{.c} + * aimath_f32_print_aiscalar(&scalar, printf); + * \endcode + */ +typedef float aiscalar_f32_t; + +/** @brief Printing a F32 tensor to console + * + * For users the function + * \code{.c} + * print_aitensor(&tensor); + * \endcode + * is prefered. + * + * @param *tensor The tensor to print. + */ +void aimath_f32_print_aitensor(const aitensor_t *tensor); + +/** @brief Printing a F32 scalar to console + * + * For users the function + * \code{.c} + * print_aiscalar(&scalar, aif32); + * \endcode + * is prefered. + * + * @param *scalar The scalar (type: float) to print. + * @param *print The print function to use + */ +void aimath_f32_print_aiscalar(const void *scalar, int (*print)(const char *format, ...)); + +/** @brief The F32 data-type indicator + * + * Use this variable to configure some element with the \link aimath_f32.h F32 \endlink data-type, + */ +extern const aimath_dtype_t *aif32; + +#endif // AIMATH_F32 diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aiopti/aiopti_adam.c b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aiopti/aiopti_adam.c new file mode 100644 index 0000000..74c7163 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aiopti/aiopti_adam.c @@ -0,0 +1,149 @@ +/** + * \file basic/base/aiopti/aiopti_adam.c + * \version 2.0alpha + * \date 12.11.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/aiopti/aiopti_adam.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_optitype_t aiopti_adam_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "ADAM", + .print_specs = aiopti_adam_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_optitype_t *aiopti_adam_type = &aiopti_adam_type_s; + +aiopti_t *aiopti_adam(aiopti_adam_t *opti) +{ + opti->base.optimizer_type = aiopti_adam_type; + opti->base.optimizer_configuration = opti; + + opti->base.sizeof_optimem = aiopti_adam_sizeof_optimem; + opti->base.init_optimem = aiopti_adam_init_optimem; + opti->base.zero_gradients = aiopti_adam_zero_gradients; + opti->base.update_params = aiopti_adam_update_params; + + return (&opti->base); +} + +uint32_t aiopti_adam_sizeof_optimem(aiopti_t *self, const aitensor_t *params){ + uint32_t memory = 0; + memory += sizeof(aiopti_adam_momentums_t); + memory += 2 * aimath_sizeof_tensor_data(params); + return memory; +} + +void aiopti_adam_init_optimem(aiopti_t *self, const aitensor_t *params, const aitensor_t *gradients, void *optimem) +{ + aiopti_adam_t *opti = (aiopti_adam_t *)(self->optimizer_configuration); + uint32_t address_counter = 0; + aiopti_adam_momentums_t *momentums = optimem; + address_counter += sizeof(aiopti_adam_momentums_t); + + momentums->m.dtype = gradients->dtype; + momentums->m.dim = gradients->dim; + momentums->m.shape = gradients->shape; + momentums->m.tensor_params = gradients->tensor_params; + momentums->m.data = optimem + address_counter; + address_counter += aimath_sizeof_tensor_data(&momentums->m); + + momentums->v.dtype = gradients->dtype; + momentums->v.dim = gradients->dim; + momentums->v.shape = gradients->shape; + momentums->v.tensor_params = gradients->tensor_params; + momentums->v.data = optimem + address_counter; + address_counter += aimath_sizeof_tensor_data(&momentums->v); + + opti->zero_tensor(&momentums->m); + opti->zero_tensor(&momentums->v); + + return; +} + +void aiopti_adam_zero_gradients(aiopti_t *self, aitensor_t *gradients) +{ + aiopti_adam_t *opti = (aiopti_adam_t *)(self->optimizer_configuration); + + opti->zero_tensor(gradients); + + return; +} + +void aiopti_adam_update_params(aiopti_t *self, aitensor_t *params, const aitensor_t *gradients, void *optimem) +{ + aiopti_adam_t *opti = (aiopti_adam_t *)(self->optimizer_configuration); + + aiopti_adam_momentums_t *momentums = optimem; + + uint8_t temp_tensor_data[aimath_sizeof_tensor_data(gradients)]; + aitensor_t temp_tensor = { + .dim = gradients->dim, + .shape = gradients->shape, + .data = temp_tensor_data, + .dtype = gradients->dtype, + .tensor_params = gradients->tensor_params + }; + + // m = beta1 * m + (1-beta1) * g + opti->scalar_mul(opti->beta1, &momentums->m, &momentums->m); + opti->scalar_mul(opti->one_minus_beta1, gradients, &temp_tensor); + opti->tensor_add(&momentums->m, &temp_tensor, &momentums->m); + + // v = beta2 * v + (1-beta2) * g*g + opti->scalar_mul(opti->beta2, &momentums->v, &momentums->v); + opti->multiply(gradients, gradients, &temp_tensor); + opti->scalar_mul(opti->one_minus_beta2, &temp_tensor, &temp_tensor); + opti->tensor_add(&momentums->v, &temp_tensor, &momentums->v); + + // lrt = lr * sqrt(1-beta2^t) / (1-beta1)^t + // params = params - lrt * m / (sqrt(v) + eps) + opti->sqrt(&momentums->v, &temp_tensor); + opti->scalar_add(opti->eps, &temp_tensor, &temp_tensor); + opti->divide(&momentums->m, &temp_tensor, &temp_tensor); + opti->scalar_mul(opti->lrt, &temp_tensor, &temp_tensor); + + temp_tensor.tensor_params = params->tensor_params; // switch params like shift to the parameter tensor params + opti->tensor_sub(params, &temp_tensor, params); + + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void aiopti_adam_print_specs(const aiopti_t *self, int (*print)(const char *format, ...)) +{ + aiopti_adam_t *self_casted = (aiopti_adam_t *) self->optimizer_configuration; + + print("lr: "); + self->dtype->print_aiscalar(self->learning_rate, print); + print("; beta1: "); + self->dtype->print_aiscalar(self_casted->beta1, print); + print("; beta2: "); + self->dtype->print_aiscalar(self_casted->beta2, print); + print("; eps: "); + self->dtype->print_aiscalar(self_casted->eps, print); + return; +} +#endif diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aiopti/aiopti_adam.h b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aiopti/aiopti_adam.h new file mode 100644 index 0000000..fb41ae3 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aiopti/aiopti_adam.h @@ -0,0 +1,295 @@ +/** + * \file basic/base/aiopti/aiopti_adam.h + * \internal + * \date 12.11.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link aiopti optimizer \endlink implementation of the [Adam optimizer](https://arxiv.org/pdf/1412.6980.pdf) + * + * This is an "abstract" data-type independent implementation. To use the optimizer, use one of the provided + * implementations for a specific hardware and data-type (for example from aiopti_adam_default.h) or set + * the required math functions on your own. + * + * The [Adam optimizer](https://arxiv.org/pdf/1412.6980.pdf) is based on SGD and uses first-order and second-order moments for adaptive estimation. + * It uses the pre-calculated gradients to optimize the given parameters. + * For every parameter \f$ p \f$ of the parameters to optimize (trainable parameters) and the related gradient \f$ g \f$ it calculates + * @f[ + * m_t = \beta_1 \cdot m_{t-1} + (1 - \beta_1) \cdot g_t + * @f] + * @f[ + * v_t = \beta_2 \cdot v_{t-1} + (1 - \beta_2) \cdot g^2_t + * @f] + * @f[ + * p_t = p_{t-1} - lr_t \cdot \frac{m_t} {\sqrt{v_t} + \hat{\epsilon}} + * @f] + * in every optimization step with \f$ lr_t = lr \cdot \frac{\sqrt{1 - \beta^t_2}} {(1 - \beta_1)^t} \f$.\n + * \f$ lr \f$ is the learning rate that defines how big the optimization steps should be, + * and therefore how fast the training will be. + * \f$ m \f$ and \f$ v \f$ are the first- and second-order moments related to the parameter and must be stored in the optimization memory + * for every parameter. + */ + +#ifndef AIOPTI_ADAM +#define AIOPTI_ADAM + +#include "core/aifes_core.h" + +typedef struct aiopti_adam aiopti_adam_t; /**< New data type name for code reduction. */ +typedef struct aiopti_adam_momentums aiopti_adam_momentums_t; /**< New data type name for code reduction. */ + +/** @brief General \link aiopti_adam.h Adam optimizer \endlink struct +* +*/ +struct aiopti_adam { + aiopti_t base; /**< Inherited field members from general optimizer struct. */ + + /** @name Optimizer configuration + * @brief Required configuration parameters for the optimizer + * + * These fields have to be configured by the user before calling the initializer function. + */ + ///@{ + void *beta1; /**< aiscalar: Exponential decay rate for the first moment estimates \f$ \beta_1 \f$ */ + void *beta2; /**< aiscalar: Exponential decay rate for the second moment estimates \f$ \beta_2 \f$ */ + void *eps; /**< aiscalar: Small positive number for numerical stability \f$ \hat{\epsilon} \f$ (avoid dividing by 0)*/ + ///@} + + /** @name Variables for internal computation + * + * These fields are automatically configured in the initializer function. + */ + ///@{ + void *beta1t; /**< aiscalar: Auxiliary variable to calculate \f$ \beta_1^t \f$ */ + void *beta2t; /**< aiscalar: Auxiliary variable to calculate \f$ \beta_2^t \f$ */ + void *one_minus_beta1; /**< aiscalar: Auxiliary variable to calculate \f$ (1 - \beta_1) \f$ */ + void *one_minus_beta2; /**< aiscalar: Auxiliary variable to calculate \f$ (1 - \beta_2) \f$ */ + void *lrt; /**< aiscalar: Auxiliary variable to calculate \f$ lr_t = lr \cdot \frac{\sqrt{1-\beta_2^t}}{(1-\beta_1^t)} \f$ */ + ///@} + + /** @name Math functions + * @brief Required data type specific math functions + */ + ///@{ + + /** @brief Required math function: Element wise tensor multiplication + * + * Requires a math function that multiplies two tensors element wise:\n + * @f[ + * result = a \odot b + * @f] + */ + void (*multiply)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + /** @brief Required math function: Element wise tensor dividation + * + * Requires a math function that devides two tensors element wise:\n + * @f[ + * result_i = a_i \div b_i + * @f] + */ + void (*divide)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + /** @brief Required math function: Element wise tensor addition + * + * Requires a math function that adds two tensors element wise:\n + * @f[ + * result = a + b + * @f] + */ + void (*tensor_add)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + /** @brief Required math function: Element wise tensor subtraction + * + * Requires a math function that subtracts two tensors element wise:\n + * @f[ + * result = a - b + * @f] + */ + void (*tensor_sub)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + /** @brief Required math function: Multiplication of a scalar with a tensor + * + * Requires a math function that multiplies a scalar with a tensor:\n + * @f[ + * result = scalar \cdot a + * @f] + */ + void (*scalar_mul)(const void *scalar, const aitensor_t *a, aitensor_t *result); + + /** @brief Required math function: Element wise addition of a scalar to a tensor + * + * Requires a math function that adds a scalar to every element of a tensor:\n + * @f[ + * result = scalar \cdot \left( \begin{array}{c} 1 & \cdots & 1 \\ \vdots & \ddots & \vdots \\ 1 & \cdots & 1 \\ \end{array}\right) + a + * @f] + */ + void (*scalar_add)(const void *scalar, const aitensor_t *a, aitensor_t *result); + + /** @brief Required math function: Square root + * + * Requires a math function that calculates the element wise square root of a tensor:\n + * @f[ + * result_{i} = \sqrt{x_i} + * @f] + */ + void (*sqrt)(const aitensor_t *x, aitensor_t *result); + + /** @brief Required math function: Sets the elements of a tensor to zero + * + * Requires a math function that sets the elements of the given tensor to zero (or to the zero_point):\n + * @f[ + * tensor_i = 0 + * @f] + */ + void (*zero_tensor)(aitensor_t *tensor); + + ///@{ +}; + +/** @brief Adam optimizer type + * + * Defines the type of the optimizer (for example for type checks and debug prints). + * See aicore_optitype for more information about the optimizer type. + */ +extern const aicore_optitype_t *aiopti_adam_type; + +/** @brief Struct for the momentum tensors of an Adam optimizer + * + * This structure is made for easy access of the momentum tensors on the optimization memory. + * + */ +struct aiopti_adam_momentums { + aitensor_t m; /**< First momentum vector. */ + aitensor_t v; /**< Second momentum vector. */ +}; + +/** @brief Initialize the given Adam optimizer + * + * This function represents the "constructor" of the abstract Adam optimizer.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example aiopti_adam_f32_default()). + * + * @param *opti The optimizer to initialize. + * @return Pointer to the (successfully) initialized general optimizer structure (aiopti_adam.base) + */ +aiopti_t *aiopti_adam(aiopti_adam_t *opti); + +/** @brief Calculates the required memory for the optimization step + * + * *Implementation of aiopti.sizeof_optimem.* + * + * Calculates the size of the memory space that must be reserved. + * The memory is used for the first and second moment tensors and is calculated by: + * + * \code 2 * (sizeof(aitensor) + sizeof(params.data)) \endcode + * + * @param *self The optimizer + * @param *params The tensor of trainable parameters to calculate the memory for + */ +uint32_t aiopti_adam_sizeof_optimem(aiopti_t *self, const aitensor_t *params); + +/** @brief Initialization of the optimization memory buffer + * + * *Implementation of aiopti.init_optimem.* + * + * Initialize the first and second moment tensors with zeros: + * @f[ + * m_{0,i} \leftarrow 0 + * @f] + * @f[ + * v_{0,i} \leftarrow 0 + * @f] + * + * Used math functions: + * * aiopti_adam.zero_tensor + * + * @param *self The optimizer + * @param *params The tensor of trainable parameters + * @param *gradients The gradients associated to the parameters + * @param *optimem The optimization memory (containing the first and second moment) associated to the parameters + */ +void aiopti_adam_init_optimem(aiopti_t *self, const aitensor_t *params, const aitensor_t *gradients, void *optimem); + +/** @brief Set the gradients to zero + * + * *Implementation of aiopti.zero_gradients.* + * + * @f[ + * g_{i} \leftarrow 0 + * @f] + * + * Used math functions: + * * aiopti_adam.zero_tensor + * + * @param *self The optimizer + * @param *gradients The gradients to set to zero + */ +void aiopti_adam_zero_gradients(aiopti_t *self, aitensor_t *gradients); + +/** @brief Update the given parameter tensor with respect to the gradients + * + * *Implementation of aiopti.update_params.* + * + * Calculate and update the values of the trainable parameters (perform one update step): + * @f[ + * m_t \leftarrow \beta_1 \cdot m_{t-1} + (1 - \beta_1) \cdot g_t + * @f] + * @f[ + * v_t \leftarrow \beta_2 \cdot v_{t-1} + (1 - \beta_2) \cdot g^2_t + * @f] + * @f[ + * p_t \leftarrow p_{t-1} - lr_t \cdot \frac{m_t} {\sqrt{v_t} + \hat{\epsilon}} + * @f] + * + * \f$ m \f$: First moment estimates\n + * \f$ v \f$: Second moment estimates\n + * \f$ p \f$: Tensor of trainable parameters to update (params)\n + * \f$ g \f$: Gradients\n + * \f$ lr \f$: Learning rate / Optimization step size\n + * \f$ \beta_1 \f$: Exponential decay rate for the first moment estimates\n + * \f$ \beta_2 \f$: Exponential decay rate for the second moment estimates\n + * \f$ \hat{\epsilon} \f$: Small positive number for numerical stability\n\n + * + * Used math functions: + * * aiopti_adam.scalar_mul + * * aiopti_adam.tensor_add + * * aiopti_adam.multiply + * * aiopti_adam.sqrt + * * aiopti_adam.scalar_add + * * aiopti_adam.divide + * * aiopti_adam.tensor_sub + * + * @param *self The optimizer + * @param *params The tensor of trainable parameters \f$ p \f$ to update + * @param *gradients The gradients \f$ g \f$ associated to the parameters + * @param *optimem The buffer to store the first and second momentums \f$ m \f$ and \f$ v \f$ + */ +void aiopti_adam_update_params(aiopti_t *self, aitensor_t *params, const aitensor_t *gradients, void *optimem); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the optimizer specification + * + * @param *self The optimizer to print the specification for + * @param *print Pointer to the print function to use + */ +void aiopti_adam_print_specs(const aiopti_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // AIOPTI_ADAM + diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aiopti/aiopti_sgd.c b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aiopti/aiopti_sgd.c new file mode 100644 index 0000000..42854fb --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aiopti/aiopti_sgd.c @@ -0,0 +1,158 @@ +/** + * \file basic/base/aiopti/aiopti_sgd.c + * \version 2.0alpha + * \date 20.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/aiopti/aiopti_sgd.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_optitype_t aiopti_sgd_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "SGD", + .print_specs = aiopti_sgd_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_optitype_t *aiopti_sgd_type = &aiopti_sgd_type_s; + +aiopti_t *aiopti_sgd(aiopti_sgd_t *opti) +{ + opti->base.optimizer_type = aiopti_sgd_type; + opti->base.optimizer_configuration = opti; + + opti->base.begin_step = 0; + opti->base.end_step = 0; + opti->base.zero_gradients = aiopti_sgd_zero_gradients; + + // Need to be set in the inherited implementation + opti->base.update_params = 0; + opti->base.sizeof_optimem = 0; + opti->base.init_optimem = 0; + + return &opti->base; +} + + +uint32_t aiopti_sgd_sizeof_optimem_with_momentum(aiopti_t *self, const aitensor_t *params){ + uint32_t memory = 0; + memory += sizeof(aitensor_t); + memory += aimath_sizeof_tensor_data(params); + return memory; +} + +uint32_t aiopti_sgd_sizeof_optimem_without_momentum(aiopti_t *self, const aitensor_t *params){ + return 0; +} + +void aiopti_sgd_init_optimem_with_momentum(aiopti_t *self, const aitensor_t *params, const aitensor_t *gradients, void *optimem) +{ + aiopti_sgd_t *opti = (aiopti_sgd_t *)(self->optimizer_configuration); + uint32_t address_counter = 0; + aitensor_t *v = (aitensor_t *) optimem; + address_counter += sizeof(aitensor_t); + + v->dtype = gradients->dtype; + v->dim = gradients->dim; + v->shape = gradients->shape; + v->tensor_params = gradients->tensor_params; + v->data = optimem + address_counter; + address_counter += aimath_sizeof_tensor_data(v); + + opti->zero_tensor(v); + + return; +} + +void aiopti_sgd_init_optimem_without_momentum(aiopti_t *self, const aitensor_t *params, const aitensor_t *gradients, void *optimem) +{ + return; +} + +void aiopti_sgd_zero_gradients(aiopti_t *self, aitensor_t *gradients) +{ + aiopti_sgd_t *opti = (aiopti_sgd_t *)(self->optimizer_configuration); + + opti->zero_tensor(gradients); + + return; +} + +void aiopti_sgd_update_params_with_momentum(aiopti_t *self, aitensor_t *params, const aitensor_t *gradients, void *optimem) +{ + aiopti_sgd_t *opti = (aiopti_sgd_t *)(self->optimizer_configuration); + + aitensor_t *v = (aitensor_t *) optimem; + + uint8_t temp_tensor_data[aimath_sizeof_tensor_data(params)]; + aitensor_t temp_tensor = { + .dim = gradients->dim, + .shape = gradients->shape, + .data = temp_tensor_data, + .dtype = gradients->dtype, + .tensor_params = gradients->tensor_params + }; + + // v = momentum * v + gradients + opti->scalar_mul(opti->momentum, v, &temp_tensor); + opti->tensor_add(&temp_tensor, gradients, v); + + // p = p - lr * v + opti->scalar_mul(opti->base.learning_rate, v, &temp_tensor); + opti->tensor_sub(params, &temp_tensor, params); + + return; +} + +void aiopti_sgd_update_params_without_momentum(aiopti_t *self, aitensor_t *params, const aitensor_t *gradients, void *optimem) +{ + aiopti_sgd_t *opti = (aiopti_sgd_t *)(self->optimizer_configuration); + + uint8_t temp_tensor_data[aimath_sizeof_tensor_data(params)]; + aitensor_t temp_tensor = { + .dim = gradients->dim, + .shape = gradients->shape, + .data = temp_tensor_data, + .dtype = gradients->dtype, + .tensor_params = gradients->tensor_params + }; + + // p = p - lr * g + opti->scalar_mul(opti->base.learning_rate, gradients, &temp_tensor); + opti->tensor_sub(params, &temp_tensor, params); + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void aiopti_sgd_print_specs(const aiopti_t *self, int (*print)(const char *format, ...)) +{ + aiopti_sgd_t *self_casted = (aiopti_sgd_t *) self->optimizer_configuration; + + print("lr: "); + self->dtype->print_aiscalar(self->learning_rate, print); + + print("; momentum: "); + self->dtype->print_aiscalar(self_casted->momentum, print); + return; +} +#endif diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aiopti/aiopti_sgd.h b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aiopti/aiopti_sgd.h new file mode 100644 index 0000000..f518278 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/base/aiopti/aiopti_sgd.h @@ -0,0 +1,272 @@ +/** + * \file basic/base/aiopti/aiopti_sgd.h + * \internal + * \date 20.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link aiopti optimizer \endlink implementation of the Stochastic Gradient Descent (with momentum) optimizer + * + * This is an "abstract" data-type independent implementation. To use the optimizer, use one of the provided + * implementations for a specific hardware and data-type (for example from aiopti_sgd_default.h) or set + * the required math functions on your own. + * + * The Stochastic Gradient Descent (SGD) optimizer is the most basic optimizer in backpropagation. + * It uses the pre-calculated gradients to optimize the given parameters. In addition, a momentum term can be configured.\n + * For every parameter \f$ p \f$ of the parameters to optimize (trainable parameters) and the related gradient \f$ g \f$ it calculates + * @f[ + * p_t = p_{t-1} - lr \cdot g_t + * @f] + * if the momentum \f$ \mu = 0 \f$ and + * @f[ + * v_t = \mu \cdot v_{t-1} + g_t + * @f] + * @f[ + * p_t = p_{t-1} - lr \cdot v_t + * @f] + * if the momentum \f$ \mu \neq 0 \f$ in every optimization step.\n + * \f$ lr \f$ is the learning rate that defines how big the optimization steps should be, + * and therefore how fast the training will be. + * \f$ v \f$ is the momentum term or velocity related to the parameter and must be stored in the optimization memory + * for every parameter when momentum is set. + */ + +#ifndef AIOPTI_SGD +#define AIOPTI_SGD + +#include "core/aifes_core.h" + +typedef struct aiopti_sgd aiopti_sgd_t; /**< New data type name for code reduction. */ + +/** @brief General \link aiopti_sgd.h Stochastic Gradient Descent (SGD) optimizer \endlink struct +* +*/ +struct aiopti_sgd { + aiopti_t base; /**< Inherited field members from general optimizer struct. */ + + /** @name Optimizer configuration + * @brief Required configuration parameters for the optimizer + * + * These fields have to be configured by the user before calling the initializer function. + */ + ///@{ + void *momentum; /**< aiscalar: Momentum(set to null to save optimization memory) */ + ///@} + + /** @name Math functions + * @brief Required data type specific math functions + */ + ///@{ + + /** @brief Required math function: Element wise tensor subtraction + * + * Requires a math function that subtracts two tensors element wise:\n + * @f[ + * result = a - b + * @f] + */ + void (*tensor_sub)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + /** @brief Required math function: Element wise tensor addition + * + * Requires a math function that adds two tensors element wise:\n + * @f[ + * result = a + b + * @f] + */ + void (*tensor_add)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + /** @brief Required math function: Multiplication of a scalar with a tensor + * + * Requires a math function that multiplies a scalar with a tensor:\n + * @f[ + * result = scalar \cdot a + * @f] + */ + void (*scalar_mul)(const void *scalar, const aitensor_t *a, aitensor_t *result); + + /** @brief Required math function: Sets the elements of a tensor to zero + * + * Requires a math function that sets the elements of the given tensor to zero (or to the zero_point):\n + * @f[ + * tensor_i = 0 + * @f] + */ + void (*zero_tensor)(aitensor_t *tensor); + + ///@} +}; + +/** @brief SGD optimizer type + * + * Defines the type of the optimizer (for example for type checks and debug prints). + * See aicore_optitype for more information about the optimizer type. + */ +extern const aicore_optitype_t *aiopti_sgd_type; + + +/** @brief Initialize the given SGD optimizer + * + * This function represents the "constructor" of the abstract SGD optimizer.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example aiopti_sgd_f32_default()). + * + * @param *opti The optimizer to initialize. + * @return Pointer to the (successfully) initialized general optimizer structure (aiopti_sgd.base) + */ +aiopti_t *aiopti_sgd(aiopti_sgd_t *opti); + +/** @brief Calculates the required memory for the optimization step when the momentum is not zero + * + * *Implementation of aiopti.sizeof_optimem.* + * + * Calculates the size of the memory space that must be reserved. + * The memory is used for the velocity tensor (momentum term) and is calculated by: + * + * \code sizeof(aitensor) + sizeof(params.data) \endcode + * + * @param *self The optimizer + * @param *params The tensor of trainable parameters to calculate the memory for + */ +uint32_t aiopti_sgd_sizeof_optimem_with_momentum(aiopti_t *self, const aitensor_t *params); + +/** @brief Calculates the required memory for the optimization step when the momentum is zero + * + * *Implementation of aiopti.sizeof_optimem.* + * + * Calculates the size of the memory space that must be reserved. + * The required memory is zero because no velocity term is needed. + * + * @param *self The optimizer + * @param *params The tensor of trainable parameters to calculae the memory for + */ +uint32_t aiopti_sgd_sizeof_optimem_without_momentum(aiopti_t *self, const aitensor_t *params); + +/** @brief Initialization of the optimization memory buffer when the momentum is not zero + * + * *Implementation of aiopti.init_optimem.* + * + * Initialize the velocity tensor with zeros: + * @f[ + * v_{0,i} \leftarrow 0 + * @f] + * + * Used math functions: + * * aiopti_sgd.zero_tensor + * + * @param *self The optimizer + * @param *params The tensor of trainable parameters + * @param *gradients The gradients associated to the parameters + * @param *optimem The optimization memory (containing the velocities) associated to the parameters + */ +void aiopti_sgd_init_optimem_with_momentum(aiopti_t *self, const aitensor_t *params, const aitensor_t *gradients, void *optimem); + +/** @brief Initialization of the optimization memory buffer when the momentum is zero + * + * *Implementation of aiopti.init_optimem.* + * + * Does nothing because no optimization memory is needed in this case. + * + * @param *self The optimizer + * @param *params The tensor of trainable parameters + * @param *gradients The gradients associated to the parameters + * @param *optimem The optimization memory associated to the parameters + */ +void aiopti_sgd_init_optimem_without_momentum(aiopti_t *self, const aitensor_t *params, const aitensor_t *gradients, void *optimem); + +/** @brief Set the gradients to zero + * + * *Implementation of aiopti.zero_gradients.* + * + * @f[ + * g_{i} \leftarrow 0 + * @f] + * + * Used math functions: + * * aiopti_sgd.zero_tensor + * + * @param *self The optimizer + * @param *gradients The gradients to set to zero + */ +void aiopti_sgd_zero_gradients(aiopti_t *self, aitensor_t *gradients); + +/** @brief Update the given parameter tensor with respect to the gradients when the momentum is not zero + * + * *Implementation of aiopti.update_params.* + * + * Calculate and update the values of the trainable parameters (perform one update step): + * @f[ + * v_t \leftarrow \mu \cdot v_{t-1} + g_t + * @f] + * @f[ + * p_t \leftarrow p_{t-1} - lr \cdot v_t + * @f] + * + * \f$ v \f$: Velocity tensor (momentum term), stored in the optimem\n + * \f$ p \f$: Tensor of trainable parameters to update (params)\n + * \f$ g \f$: Gradients\n + * \f$ lr \f$: Learning rate / Optimization step size\n + * \f$ \mu \f$: Momentum\n\n + * + * Used math functions: + * * aiopti_sgd.scalar_mul + * * aiopti_sgd.tensor_add + * * aiopti_sgd.tensor_sub + * + * @param *self The optimizer + * @param *params The tensor of trainable parameters \f$ p \f$ to update + * @param *gradients The gradients \f$ g \f$ associated to the parameters + * @param *optimem The buffer to store the velocity \f$ v \f$ + */ +void aiopti_sgd_update_params_with_momentum(aiopti_t *self, aitensor_t *params, const aitensor_t *gradients, void *optimem); + +/** @brief Update the given parameter tensor with respect to the gradients when the momentum is zero + * + * *Implementation of aiopti.update_params.* + * + * Calculate and update the values of the trainable parameters (perform one update step): + * @f[ + * p_t \leftarrow p_{t-1} - lr \cdot g_t + * @f] + * + * Used math functions: + * * aiopti_sgd.scalar_mul + * * aiopti_sgd.tensor_sub + * + * \f$ p \f$: Tensor of trainable parameters to update (params)\n + * \f$ g \f$: Gradients\n + * \f$ lr \f$: Learning rate / Optimization step size\n\n + * + * @param *self The optimizer + * @param *params The tensor of trainable parameters \f$ p \f$ to update + * @param *gradients The gradients \f$ g \f$ associated to the parameters + * @param *optimem Not required because no velocity is stored + */ +void aiopti_sgd_update_params_without_momentum(aiopti_t *self, aitensor_t *params, const aitensor_t *gradients, void *optimem); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the optimizer specification + * + * @param *self The optimizer to print the specification for + * @param *print Pointer to the print function to use + */ +void aiopti_sgd_print_specs(const aiopti_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // AIOPTI_SGD + diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/cmsis/ailayer/ailayer_dense_cmsis.c b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/cmsis/ailayer/ailayer_dense_cmsis.c new file mode 100644 index 0000000..a978db1 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/cmsis/ailayer/ailayer_dense_cmsis.c @@ -0,0 +1,49 @@ +/** + * \file basic/cmsis/ailayer/ailayer_dense_cmsis.c + * \version 2.0alpha + * \date 15.03.2021 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/ailayer/ailayer_dense_default.h" +#include "../../../aifes.h" + +#if __arm__ +#ifdef AIFES_WITH_CMSIS + +#include "basic/cmsis/ailayer/ailayer_dense_cmsis.h" +#include "basic/cmsis/aimath/aimath_f32_cmsis.h" + + +ailayer_t *ailayer_dense_f32_cmsis(ailayer_dense_t *layer, ailayer_t *input_layer) +{ + layer->result_dtype = aif32; + layer->weights_dtype = aif32; + layer->bias_dtype = aif32; + + layer->linear = aimath_f32_cmsis_linear; + layer->mat_mul = aimath_f32_cmsis_mat_mul; + layer->tensor_add = aimath_f32_default_tensor_add; + + return ailayer_dense(layer, input_layer); +} + +#endif // AIFES_WITH_CMSIS +#endif //__arm__ diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/cmsis/ailayer/ailayer_dense_cmsis.h b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/cmsis/ailayer/ailayer_dense_cmsis.h new file mode 100644 index 0000000..4945e23 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/cmsis/ailayer/ailayer_dense_cmsis.h @@ -0,0 +1,77 @@ +/** + * \file basic/cmsis/ailayer/ailayer_dense_cmsis.h + * \internal + * \date 15.03.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief [Arm CMSIS](https://developer.arm.com/tools-and-software/embedded/cmsis) implementation of the \link ailayer_dense.h Dense layer \endlink for Arm Cortex processors. + * + * Arm CMSIS implementations of the Dense layer in \link aimath_f32.h F32 \endlink data-type. + * These implementations are specifically designed for the Arm Cortex processors and take advantage of SIMD instructions. + * For more information about the Dense layer refer to ailayer_dense.h. + */ + +#ifndef AILAYER_DENSE_CMSIS +#define AILAYER_DENSE_CMSIS + +#include "../../../aifes.h" + +#if __arm__ +#ifdef AIFES_WITH_CMSIS + +#include "basic/base/ailayer/ailayer_dense.h" + +#include "basic/cmsis/aimath/aimath_f32_cmsis.h" + +/** @brief Initializes and connect a Dense layer with the \link aimath_f32.h F32 \endlink CMSIS implementation + * + * Example: Create the layer structure with pretrained weights:\n + * \code{.c} + * // Use constant data only for inference. For training remove the const qualifier!! + * const float weights_data_dense[] = {-10.1164f, -8.4212f, 5.4396f, 7.297f, -7.6482f, -9.0155f}; + * const float bias_data_dense[] = {-2.9653f, 2.3677f, -1.5968f}; + * ailayer_dense_f32_t dense_layer = { + * .neurons = 3, + * .weights.data = (float *) weights_data_dense, + * .bias.data = (float *) bias_data_dense + * }; + * \endcode + * + * Example: Create the layer structure for training:\n + * \code{.c} + * ailayer_dense_f32_t dense_layer = { + * .neurons = 3 + * }; + * \endcode + * + * Example: Initialize and connect the layer:\n + * \code{.c} + * x = ailayer_dense_f32_cmsis(&dense_layer, x); + * \endcode + * + * @param *layer The layer structure to initialize. + * @param *input_layer The prior layer. + * @return The (successfully) initialized layer structure. + */ +ailayer_t *ailayer_dense_f32_cmsis(ailayer_dense_t *layer, ailayer_t *input_layer); + +#endif // AILAYER_DENSE_CMSIS + +#endif // AIFES_WITH_CMSIS +#endif //__arm__ diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/cmsis/aimath/aimath_f32_cmsis.c b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/cmsis/aimath/aimath_f32_cmsis.c new file mode 100644 index 0000000..5f8a2bc --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/cmsis/aimath/aimath_f32_cmsis.c @@ -0,0 +1,104 @@ +/** + * \file basic/cmsis/aimath/aimath_f32_cmsis.c + * \version 2.0alpha + * \date 16.03.2021 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + + +#include "basic/cmsis/aimath/aimath_f32_cmsis.h" +#include "../../../aifes.h" + +#if __arm__ +#ifdef AIFES_WITH_CMSIS +#include "arm_math.h" + +/** +* Math CMSIS Matrix Multiplication and boradtcast Ass +* +* Matrixmultiplication using the CMSIS DSP Library and broadcast add using for loop +* +*/ +void aimath_f32_cmsis_linear(const aitensor_t *a, const aitensor_t *b, const aitensor_t *c, aitensor_t *result) +{ + +#ifdef SHAPE_CHECK + if(a->shape[1] != b->shape[0]) + { + LOG_E("MatMul input shapes doesn't match.\n"); + return; + } + if(a->shape[0] != result->shape[0] || b->shape[1] != result->shape[1]) + { + LOG_E("MatMul output shape doesn't match.\n"); + return; + } +#endif + + uint16_t i, j; + + float *c_data = c != 0 ? (float *) c->data : 0; + float *result_data = (float *) result->data; + + + aimath_f32_cmsis_mat_mul(a, b, result); + + + for(i = 0; i < result->shape[0]; i++) + { + for(j = 0; j < result->shape[1]; j++) + { + result_data[i*result->shape[1] + j] += c_data[j]; + } + } + + + + return; +} + + +/** +* Math CMSIS Matrix Multiplication +* +* Matrixmultiplication using the CMSIS DSP Library +* +* \ref https://arm-software.github.io/CMSIS_5/DSP/html/group__MatrixMult.html +*/ +void aimath_f32_cmsis_mat_mul(const aitensor_t *a, const aitensor_t *b, aitensor_t *result) +{ + float *a_data = (float *) a->data; + float *b_data = (float *) b->data; + float *result_data = (float *) result->data; + + + arm_matrix_instance_f32 a_mat; /* Matrix a Instance */ + arm_matrix_instance_f32 b_mat; /* Matrix b Instance */ + arm_matrix_instance_f32 result_mat; /* Matrix result Instance */ + + arm_mat_init_f32(&a_mat, a->shape[0], a->shape[1], a_data); + arm_mat_init_f32(&b_mat, b->shape[0], b->shape[1], b_data); + arm_mat_init_f32(&result_mat, result->shape[0], result->shape[1], result_data); + + + arm_mat_mult_f32(&a_mat, &b_mat, &result_mat); +} +#endif // AIFES_WITH_CMSIS +#endif // __arm__ \ No newline at end of file diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/cmsis/aimath/aimath_f32_cmsis.h b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/cmsis/aimath/aimath_f32_cmsis.h new file mode 100644 index 0000000..199bab1 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/cmsis/aimath/aimath_f32_cmsis.h @@ -0,0 +1,130 @@ +/** + * \file basic/cmsis/aimath/aimath_f32_cmsis.h + * \internal + * \date 15.03.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ +#ifndef AIMATH_F32_CMSIS +#define AIMATH_F32_CMSIS + + +#include "basic/base/aimath/aimath_f32.h" +#include "../../../aifes.h" + +#if __arm__ +#ifdef AIFES_WITH_CMSIS + + +/** @brief Performs a matrix multiplication of f32 tensors a and b and adds a 1D tensor c to each row, using the ARM CMSIS DSP + * + * @details The addition of the horizontal vector c is performed via broadcast, i.e. elementwise in each column + * Mathematically this broadcast is equal to multiplying c with an vertical vector of ones (with the same number of elements as c) + * + * @details + * and adding the result to a * b + * @f[ + * result = a \cdot b + \left( \begin{array}{r} + 1 \\ + 1 \\ + \vdots \\ + 1 \\ + \end{array}\right) \cdot c + * @f] + * + * Example: + * @f[ + * a = \left( \begin{array}{rrr} + 1 & 2 & 3 \\ + 4 & 5 & 6 \\ + 7 & 8 & 9 + \end{array}\right) + * @f] + * + * @f[ + * b = \left( \begin{array}{rr} + 1 & 0 \\ + 0 & 1 \\ + 0 & 0 + \end{array}\right) + * @f] + * + * @f[ + * c = \left( \begin{array}{rr} + 2 & 5 + \end{array}\right) + * @f] + * + * @f[ + * result = a \cdot b + \left( \begin{array}{r} + 1 \\ + 1 \\ + 1 \\ + \end{array}\right) \cdot c + * @f] + * + * @f[ + * = \left( \begin{array}{rr} + 1 & 2 \\ + 4 & 5 \\ + 7 & 8 + \end{array}\right) + + + \left( \begin{array}{rr} + 2 & 5 \\ + 2 & 5 \\ + 2 & 5 + \end{array}\right) + * @f] + * + * @details + * + * @f[ + * = \left( \begin{array}{rr} + 3 & 7 \\ + 6 & 10 \\ + 9 & 13 + \end{array}\right) + * @f] + * + * @param *a f32 tensor a + * @param *b f32 tensor b + * @param *c Tensor c, 1 row and as many columns as the result + * @param *result Result f32 tensor + */ +void aimath_f32_cmsis_linear(const aitensor_t *a, const aitensor_t *b, const aitensor_t *c, aitensor_t *result); + +/** @brief Performs a matrix multiplication of f32 tensors a and b, using the ARM CMSIS DSP + * + * @details + * @f[ + * result = a \cdot b + * @f] + * + * @param *a f32 tensor a + * @param *b f32 tensor b + * @param *result Result f32 tensor of the multiplication + */ +void aimath_f32_cmsis_mat_mul(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + +#endif // AIFES_WITH_CMSIS +#endif //__arm__ +#endif // AIMATH_F32_CMSIS diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_dense_default.c b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_dense_default.c new file mode 100644 index 0000000..96d6e85 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_dense_default.c @@ -0,0 +1,39 @@ +/** + * \file basic/default/ailayer/ailayer_dense_default.c + * \version 2.0alpha + * \date 28.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief See ailayer_dense_default.h for documentation. + * \details + */ + +#include "basic/default/ailayer/ailayer_dense_default.h" + + +ailayer_t *ailayer_dense_f32_default(ailayer_dense_f32_t *layer, ailayer_t *input_layer) +{ + layer->result_dtype = aif32; + layer->weights_dtype = aif32; + layer->bias_dtype = aif32; + + layer->linear = aimath_f32_default_linear; + layer->mat_mul = aimath_f32_default_mat_mul; + layer->tensor_add = aimath_f32_default_tensor_add; + + return ailayer_dense(layer, input_layer); +} diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_dense_default.h b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_dense_default.h new file mode 100644 index 0000000..26c31f2 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_dense_default.h @@ -0,0 +1,69 @@ +/** + * \file basic/default/ailayer/ailayer_dense_default.h + * \internal + * \date 28.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link ailayer_dense.h Dense layer \endlink + * + * Hardware independent implementations of the Dense layer in \link aimath_f32.h F32 \endlink data-type. + * For more information about the Dense layer refer to ailayer_dense.h. + */ + +#ifndef AILAYER_DENSE_DEFAULT +#define AILAYER_DENSE_DEFAULT + +#include "basic/base/ailayer/ailayer_dense.h" +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct ailayer_dense ailayer_dense_f32_t; + +/** @brief Initializes and connect a \link ailayer_dense.h Dense layer \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * Example: Create the layer structure with pretrained weights:\n + * \code{.c} + * // Use constant data only for inference. For training remove the const qualifier!! + * const float weights_data_dense[] = {-10.1164f, -8.4212f, 5.4396f, 7.297f, -7.6482f, -9.0155f}; + * const float bias_data_dense[] = {-2.9653f, 2.3677f, -1.5968f}; + * ailayer_dense_f32_t dense_layer = { + * .neurons = 3, + * .weights.data = (float *) weights_data_dense, + * .bias.data = (float *) bias_data_dense + * }; + * \endcode + * + * Example: Create the layer structure for training:\n + * \code{.c} + * ailayer_dense_f32_t dense_layer = { + * .neurons = 3 + * }; + * \endcode + * + * Example: Initialize and connect the layer:\n + * \code{.c} + * x = ailayer_dense_f32_default(&dense_layer, x); + * \endcode + * + * @param *layer The layer structure to initialize. + * @param *input_layer The prior layer. + * @return The (successfully) initialized layer structure. + */ +ailayer_t *ailayer_dense_f32_default(ailayer_dense_f32_t *layer, ailayer_t *input_layer); + +#endif // AILAYER_DENSE_DEFAULT diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_elu_default.c b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_elu_default.c new file mode 100644 index 0000000..d6b65ef --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_elu_default.c @@ -0,0 +1,40 @@ +/** + * \file basic/default/ailayer/ailayer_elu_default.c + * \version 2.0alpha + * \date 16.03.2021 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/ailayer/ailayer_elu_default.h" + +ailayer_t *ailayer_elu_f32_default(ailayer_elu_f32_t *layer, ailayer_t *input_layer) +{ + layer->base.dtype = aif32; + layer->base.alpha = &(layer->alpha); + + //forward + layer->base.elu = aimath_f32_default_elu; + + // backward + layer->base.d_elu = aimath_f32_default_d_elu; + layer->base.multiply = aimath_f32_default_multiply; + + return ailayer_elu(&layer->base, input_layer); +} diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_elu_default.h b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_elu_default.h new file mode 100644 index 0000000..e1548c4 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_elu_default.h @@ -0,0 +1,66 @@ +/** + * \file basic/default/ailayer/ailayer_elu_default.h + * \internal + * \date 16.03.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link ailayer_elu.h ELU layer \endlink + * + * Hardware independent implementations of the ELU layer in \link aimath_f32.h F32 \endlink data-type. + * For more information about the ELU layer refer to ailayer_elu.h. + */ +#ifndef AILAYER_ELU_DEFAULT +#define AILAYER_ELU_DEFAULT + +#include "basic/base/ailayer/ailayer_elu.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct ailayer_elu_f32 ailayer_elu_f32_t; + +/** @brief Data-type specific ELU layer struct for \link aimath_f32.h F32 \endlink + * + * Adds a data field for the constant alpha in \link aimath_f32.h F32 \endlink to the base implementation. + */ +struct ailayer_elu_f32 { + ailayer_elu_t base; /**< Inherited field members from general ailayer_elu struct. */ + aiscalar_f32_t alpha; /**< Data-type specific parameter used to calculate ELU function for input values < 0. */ +}; + +/** @brief Initializes and connect an \link ailayer_elu.h ELU layer \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * Example: Create the layer structure:\n + * \code{.c} + * ailayer_elu_f32_t elu_layer = { + * .alpha = 1.0f + * }; + * \endcode + * + * Example: Initialize and connect the layer:\n + * \code{.c} + * x = ailayer_elu_f32_default(&elu_layer, x); + * \endcode + * + * @param *layer The layer structure to initialize. + * @param *input_layer The prior layer. + * @return The (successfully) initialized layer structure. + */ +ailayer_t *ailayer_elu_f32_default(ailayer_elu_f32_t *layer, ailayer_t *input_layer); + +#endif // AILAYER_ELU_DEFAULT diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_input_default.c b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_input_default.c new file mode 100644 index 0000000..5d77f44 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_input_default.c @@ -0,0 +1,32 @@ +/** + * \file basic/default/ailayer/ailayer_input_default.c + * \version 2.0alpha + * \date 12.11.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/ailayer/ailayer_input_default.h" + +ailayer_t *ailayer_input_f32_default(ailayer_input_f32_t *layer) +{ + layer->dtype = aif32; + + return ailayer_input(layer); +} diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_input_default.h b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_input_default.h new file mode 100644 index 0000000..669fa47 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_input_default.h @@ -0,0 +1,59 @@ +/** + * \file basic/default/ailayer/ailayer_input_default.h + * \internal + * \date 19.11.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link ailayer_input.h Input layer \endlink + * + * Hardware independent implementations of the Input layer in \link aimath_f32.h F32 \endlink data-type. + * For more information about the Input layer refer to ailayer_input.h. + */ + +#ifndef AILAYER_INPUT_DEFAULT +#define AILAYER_INPUT_DEFAULT + +#include "basic/base/ailayer/ailayer_input.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct ailayer_input ailayer_input_f32_t; + +/** @brief Initializes and connect an \link ailayer_input.h Input layer \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * Example: Create the layer structure:\n + * \code{.c} + * uint16_t input_layer_shape[] = {1, 2}; + * ailayer_input_f32_t input_layer = { + * .input_dim = 2, + * .input_shape = input_layer_shape + * }; + * \endcode + * + * Example: Initialize and connect the layer:\n + * \code{.c} + * x = ailayer_input_f32_default(&input_layer); + * \endcode + * + * @param *layer The layer structure to initialize. + * @return The (successfully) initialized layer structure. + */ +ailayer_t *ailayer_input_f32_default(ailayer_input_f32_t *layer); + +#endif // AILAYER_INPUT_DEFAULT diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_leaky_relu_default.c b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_leaky_relu_default.c new file mode 100644 index 0000000..4219579 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_leaky_relu_default.c @@ -0,0 +1,40 @@ +/** + * \file basic/default/ailayer/ailayer_leaky_relu_default.c + * \version 2.0alpha + * \date 16.03.2021 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/ailayer/ailayer_leaky_relu_default.h" + +ailayer_t *ailayer_leaky_relu_f32_default(ailayer_leaky_relu_f32_t *layer, ailayer_t *input_layer) +{ + layer->base.dtype = aif32; + layer->base.alpha = &(layer->alpha); + + //forward + layer->base.leaky_relu = aimath_f32_default_leaky_relu; + + // backward + layer->base.d_leaky_relu = aimath_f32_default_d_leaky_relu; + layer->base.multiply = aimath_f32_default_multiply; + + return ailayer_leaky_relu(&layer->base, input_layer); +} diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_leaky_relu_default.h b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_leaky_relu_default.h new file mode 100644 index 0000000..e0199bd --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_leaky_relu_default.h @@ -0,0 +1,68 @@ + +/** + * \file basic/default/ailayer/ailayer_leaky_relu_default.h + * \internal + * \date 16.03.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link ailayer_leaky_relu.h Leaky ReLU layer \endlink + * + * Hardware independent implementations of the Leaky ReLU layer in \link aimath_f32.h F32 \endlink data-type. + * For more information about the Leaky ReLU layer refer to ailayer_leaky_relu.h. + */ + +#ifndef AILAYER_LEAKY_RELU_DEFAULT +#define AILAYER_LEAKY_RELU_DEFAULT + +#include "basic/base/ailayer/ailayer_leaky_relu.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct ailayer_leaky_relu_f32 ailayer_leaky_relu_f32_t; + +/** @brief Data-type specific Leaky ReLU layer struct for \link aimath_f32.h F32 \endlink + * + * Adds a data field for the constant alpha in \link aimath_f32.h F32 \endlink to the base implementation. + */ +struct ailayer_leaky_relu_f32 { + ailayer_leaky_relu_t base; /**< Inherited field members from general ailayer_leaky_relu struct. */ + aiscalar_f32_t alpha; /**< Data type-specific parameter used to calculate Leaky ReLU function for input values < 0. */ +}; + +/** @brief Initializes and connect a \link ailayer_leaky_relu.h Leaky ReLU layer \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * Example: Create the layer structure:\n + * \code{.c} + * ailayer_leaky_relu_f32_t leaky_relu_layer = { + * .alpha = 0.01f + * }; + * \endcode + * + * Example: Initialize and connect the layer:\n + * \code{.c} + * x = ailayer_leaky_relu_f32_default(&leaky_relu_layer, x); + * \endcode + * + * @param *layer the layer structure to be initialized + * @param *input_layer the prior layer that provides the input to this Leaky ReLU layer + * @return the initialized Leaky ReLU layer structure + */ +ailayer_t *ailayer_leaky_relu_f32_default(ailayer_leaky_relu_f32_t *layer, ailayer_t *input_layer); + +#endif // AILAYER_LEAKY_RELU_DEFAULT diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_relu_default.c b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_relu_default.c new file mode 100644 index 0000000..56b2b2d --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_relu_default.c @@ -0,0 +1,39 @@ +/** + * \file basic/default/ailayer/ailayer_relu_default.c + * \version 2.0alpha + * \date 07.12.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/ailayer/ailayer_relu_default.h" + +ailayer_t *ailayer_relu_f32_default(ailayer_relu_f32_t *layer, ailayer_t *input_layer) +{ + layer->dtype = aif32; + + //forward + layer->relu = aimath_f32_default_relu; + + // backward + layer->d_relu = aimath_f32_default_d_relu; + layer->multiply = aimath_f32_default_multiply; + + return ailayer_relu(layer, input_layer); +} diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_relu_default.h b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_relu_default.h new file mode 100644 index 0000000..83e2723 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_relu_default.h @@ -0,0 +1,56 @@ +/** + * \file basic/default/ailayer/ailayer_relu_default.h + * \internal + * \date 07.12.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link ailayer_relu.h ReLU layer \endlink + * + * Hardware independent implementations of the ReLU layer in \link aimath_f32.h F32 \endlink data-type. + * For more information about the ReLU layer refer to ailayer_relu.h. + */ + +#ifndef AILAYER_RELU_DEFAULT +#define AILAYER_RELU_DEFAULT + +#include "basic/base/ailayer/ailayer_relu.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct ailayer_relu ailayer_relu_f32_t; + +/** @brief Initializes and connect a \link ailayer_relu.h ReLU layer \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * Example: Create the layer structure:\n + * \code{.c} + * ailayer_relu_f32_t relu_layer; + * \endcode + * + * Example: Initialize and connect the layer:\n + * \code{.c} + * x = ailayer_relu_f32_default(&relu_layer, x); + * \endcode + * + * @param *layer The layer structure to initialize. + * @param *input_layer The prior layer. + * @return The (successfully) initialized layer structure. + */ +ailayer_t *ailayer_relu_f32_default(ailayer_relu_f32_t *layer, ailayer_t *input_layer); + +#endif // AILAYER_RELU_DEFAULT diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_sigmoid_default.c b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_sigmoid_default.c new file mode 100644 index 0000000..2ed7434 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_sigmoid_default.c @@ -0,0 +1,64 @@ +/** + * \file basic/default/ailayer/ailayer_sigmoid_default.c + * \version 2.0alpha + * \date 12.11.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/ailayer/ailayer_sigmoid_default.h" + +ailayer_t *ailayer_sigmoid_f32_default(ailayer_sigmoid_f32_t *layer, ailayer_t *input_layer) +{ + layer->dtype = aif32; + + //forward + layer->sigmoid = aimath_f32_default_sigmoid; + + // backward + layer->d_sigmoid = aimath_f32_default_d_sigmoid; + layer->multiply = aimath_f32_default_multiply; + + layer->base.get_result_bound = ailayer_sigmoid_get_result_bound_f32_default; + + return ailayer_sigmoid(layer, input_layer); +} + +uint8_t ailayer_sigmoid_get_result_bound_f32_default(const ailayer_t *self, const uint8_t selector, void *result_bound) +{ + float *bound = (float *) result_bound; + + switch(selector){ + case AILAYER_RESULT_LOWER_BOUND: + *bound = 0.0f; + return TRUE; + case AILAYER_RESULT_UPPER_BOUND: + *bound = 1.0f; + return TRUE; + case AILAYER_DELTAS_LOWER_BOUND: + return FALSE; + case AILAYER_DELTAS_UPPER_BOUND: + return FALSE; + default: + #ifdef AIDEBUG_PRINT_ERROR_MESSAGES + printf("\n+++ ERROR: Not defined result bound selector.\n"); + #endif // AIDEBUG_PRINT_ERROR_MESSAGES + return FALSE; + } +} diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_sigmoid_default.h b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_sigmoid_default.h new file mode 100644 index 0000000..1ee255f --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_sigmoid_default.h @@ -0,0 +1,73 @@ +/** + * \file basic/default/ailayer/ailayer_sigmoid_default.h + * \internal + * \date 12.11.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link ailayer_sigmoid.h Sigmoid layer \endlink + * + * Hardware independent implementations of the Sigmoid layer in \link aimath_f32.h F32 \endlink data-type. + * For more information about the Sigmoid layer refer to ailayer_sigmoid.h. + */ + +#ifndef AILAYER_SIGMOID_DEFAULT +#define AILAYER_SIGMOID_DEFAULT + +#include "basic/base/ailayer/ailayer_sigmoid.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct ailayer_sigmoid ailayer_sigmoid_f32_t; + +/** @brief Initializes and connect a \link ailayer_sigmoid.h Sigmoid layer \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * Example: Create the layer structure:\n + * \code{.c} + * ailayer_sigmoid_f32_t sigmoid_layer; + * \endcode + * + * Example: Initialize and connect the layer:\n + * \code{.c} + * x = ailayer_sigmoid_f32_default(&sigmoid_layer, x); + * \endcode + * + * @param *layer The layer structure to initialize. + * @param *input_layer The prior layer. + * @return The (successfully) initialized layer structure. + */ +ailayer_t *ailayer_sigmoid_f32_default(ailayer_sigmoid_f32_t *layer, ailayer_t *input_layer); + +/** @brief Get the value bounds for Sigmoid layers + * + * For the float 32 representation only result bounds are defined. + * + * @param *self Referenced layer structure. + * @param selector Select, which bound value to get. Possible values: + * - AILAYER_RESULT_LOWER_BOUND + * - AILAYER_RESULT_UPPER_BOUND + * - AILAYER_DELTAS_LOWER_BOUND + * - AILAYER_DELTAS_UPPER_BOUND + * @param result_bound Pointer to the memory space, the function writes the requested value to. + * @return + * - TRUE: requested value available and set to result_bound + * - FALSE: requestet value is _not_ available and result_bound is __not__ set + */ +uint8_t ailayer_sigmoid_get_result_bound_f32_default(const ailayer_t *self, const uint8_t selector, void *result_bound); + +#endif // AILAYER_SIGMOID_DEFAULT diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_softmax_default.c b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_softmax_default.c new file mode 100644 index 0000000..d393e3d --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_softmax_default.c @@ -0,0 +1,35 @@ +/** + * \file basic/default/ailayer/ailayer_softmax_default.c + * \version 2.0alpha + * \date 07.12.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/ailayer/ailayer_softmax_default.h" + +ailayer_t *ailayer_softmax_f32_default(ailayer_softmax_f32_t *layer, ailayer_t *input_layer) +{ + layer->dtype = aif32; + + //forward + layer->softmax = aimath_f32_default_softmax; + + return ailayer_softmax(layer, input_layer); +} diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_softmax_default.h b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_softmax_default.h new file mode 100644 index 0000000..cbb95c4 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_softmax_default.h @@ -0,0 +1,56 @@ +/** + * \file basic/default/ailayer/ailayer_softmax_default.h + * \internal + * \date 07.12.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link ailayer_softmax.h Softmax layer \endlink + * + * Hardware independent implementations of the Softmax layer in \link aimath_f32.h F32 \endlink data-type. + * For more information about the Softmax layer refer to ailayer_softmax.h. + */ + +#ifndef AILAYER_SOFTMAX_DEFAULT +#define AILAYER_SOFTMAX_DEFAULT + +#include "basic/base/ailayer/ailayer_softmax.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct ailayer_softmax ailayer_softmax_f32_t; + +/** @brief Initializes and connect an \link ailayer_softmax.h Softmax layer \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * Example: Create the layer structure:\n + * \code{.c} + * ailayer_softmax_f32_t softmax_layer; + * \endcode + * + * Example: Initialize and connect the layer:\n + * \code{.c} + * x = ailayer_softmax_f32_default(&softmax_layer, x); + * \endcode + * + * @param *layer The layer structure to initialize. + * @param *input_layer The prior layer. + * @return The (successfully) initialized layer structure. + */ +ailayer_t *ailayer_softmax_f32_default(ailayer_softmax_f32_t *layer, ailayer_t *input_layer); + +#endif // AILAYER_SOFTMAX_DEFAULT diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_softsign_default.c b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_softsign_default.c new file mode 100644 index 0000000..a766196 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_softsign_default.c @@ -0,0 +1,39 @@ +/** + * \file basic/default/ailayer/ailayer_softsign_default.c + * \version 2.0alpha + * \date 16.03.2021 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/ailayer/ailayer_softsign_default.h" + +ailayer_t *ailayer_softsign_f32_default(ailayer_softsign_f32_t *layer, ailayer_t *input_layer) +{ + layer->dtype = aif32; + + //forward + layer->softsign = aimath_f32_default_softsign; + + // backward + layer->d_softsign = aimath_f32_default_d_softsign; + layer->multiply = aimath_f32_default_multiply; + + return ailayer_softsign(layer, input_layer); +} diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_softsign_default.h b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_softsign_default.h new file mode 100644 index 0000000..06637d8 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_softsign_default.h @@ -0,0 +1,57 @@ +/** + * \file basic/default/ailayer/ailayer_softsign_default.h + * \internal + * \date 16.03.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link ailayer_softsign.h Softsign layer \endlink + * + * Hardware independent implementations of the Softsign layer in \link aimath_f32.h F32 \endlink data-type. + * For more information about the Softsign layer refer to ailayer_softsign.h. + */ + + +#ifndef AILAYER_SOFTSIGN_DEFAULT +#define AILAYER_SOFTSIGN_DEFAULT + +#include "basic/base/ailayer/ailayer_softsign.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct ailayer_softsign ailayer_softsign_f32_t; + +/** @brief Initializes and connect a \link ailayer_softsign.h Softsign layer \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * Example: Create the layer structure:\n + * \code{.c} + * ailayer_softsign_f32_t softsign_layer; + * \endcode + * + * Example: Initialize and connect the layer:\n + * \code{.c} + * x = ailayer_softsign_f32_default(&softsign_layer, x); + * \endcode + * + * @param *layer The layer structure to initialize. + * @param *input_layer The prior layer. + * @return The (successfully) initialized layer structure. + */ +ailayer_t *ailayer_softsign_f32_default(ailayer_softsign_f32_t *layer, ailayer_t *input_layer); + +#endif // AILAYER_SOFTSIGN_DEFAULT diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_tanh_default.c b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_tanh_default.c new file mode 100644 index 0000000..87c3fe6 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_tanh_default.c @@ -0,0 +1,67 @@ + +/** + * \file basic/default/ailayer/ailayer_tanh_default.c + * \version 2.0alpha + * \date 17.03.2021 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/ailayer/ailayer_tanh_default.h" + +ailayer_t *ailayer_tanh_f32_default(ailayer_tanh_f32_t *layer, ailayer_t *input_layer) +{ + layer->dtype = aif32; + + //forward + layer->tanh = aimath_f32_default_tanh; + + // backward + layer->d_tanh = aimath_f32_default_d_tanh; + layer->multiply = aimath_f32_default_multiply; + + layer->base.get_result_bound = ailayer_tanh_get_result_bound_f32_default; + + return ailayer_tanh(layer, input_layer); +} + +uint8_t ailayer_tanh_get_result_bound_f32_default(const ailayer_t *self, const uint8_t selector, void *result_bound) +{ + float *bound = (float *) result_bound; + + switch(selector){ + case AILAYER_RESULT_LOWER_BOUND: + *bound = -1.0f; + return TRUE; + case AILAYER_RESULT_UPPER_BOUND: + *bound = 1.0f; + return TRUE; + case AILAYER_DELTAS_LOWER_BOUND: + return FALSE; + case AILAYER_DELTAS_UPPER_BOUND: + return FALSE; + default: + #ifdef AIDEBUG_PRINT_ERROR_MESSAGES + printf("\n+++ ERROR: Not defined result bound selector.\n"); + #endif // AIDEBUG_PRINT_ERROR_MESSAGES + return FALSE; + } +} + + diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_tanh_default.h b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_tanh_default.h new file mode 100644 index 0000000..9e31ea8 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailayer/ailayer_tanh_default.h @@ -0,0 +1,70 @@ + +/** + * \file basic/default/ailayer/ailayer_tanh_default.h + * \internal + * \date 17.03.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link ailayer_tanh.h Tanh layer \endlink + * + * Hardware independent implementations of the Tanh layer in \link aimath_f32.h F32 \endlink data-type. + * For more information about the Tanh layer refer to ailayer_tanh.h. + */ + +#ifndef AILAYER_TANH_DEFAULT +#define AILAYER_TANH_DEFAULT + +#include "basic/base/ailayer/ailayer_tanh.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct ailayer_tanh ailayer_tanh_f32_t; + +/** @brief Initializes and connect an \link ailayer_tanh.h Tanh layer \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * Example: Create the layer structure:\n + * \code{.c} + * ailayer_tanh_f32_t tanh_layer; + * \endcode + * + * Example: Initialize and connect the layer:\n + * \code{.c} + * x = ailayer_tanh_f32_default(&tanh_layer, x); + * \endcode + * + * Initializes the tanh layer structure with the float 32 default implementation + * + * @param *layer The layer structure to initialize. + * @param *input_layer The prior layer. + * @return The (successfully) initialized layer structure. + */ +ailayer_t *ailayer_tanh_f32_default(ailayer_tanh_f32_t *layer, ailayer_t *input_layer); + +/** @brief Get valid value range of output of tanh layer +* +* Gives the upper and lower bound of the results obtained by interference with the tanh activation function +* +* @param *self the layer to get the upper and lower limit for +* @param *selector the type of the bound (result/deltas, upper/lower) +* @param *result_bound value of the limit, if exists +* @return TRUE or FALSE in case no limit exists +*/ +uint8_t ailayer_tanh_get_result_bound_f32_default(const ailayer_t *self, const uint8_t selector, void *result_bound); + +#endif // AILAYER_TANH_DEFAULT diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailoss/ailoss_crossentropy_default.c b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailoss/ailoss_crossentropy_default.c new file mode 100644 index 0000000..0a0dd99 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailoss/ailoss_crossentropy_default.c @@ -0,0 +1,48 @@ +/** + * \file basic/default/ailoss/ailoss_crossentropy_default.c + * \version 2.0alpha + * \date 14.01.2021 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/ailoss/ailoss_crossentropy_default.h" + + +ailoss_t *ailoss_crossentropy_f32_default(ailoss_crossentropy_f32_t *loss, ailayer_t *input_layer) +{ + loss->dtype = aif32; + + loss->tensor_sub = aimath_f32_default_tensor_sub; + + // Automatically set the right crossentropy function for the output layer. + if(input_layer->layer_type == ailayer_sigmoid_type){ + loss->crossentropy = aimath_f32_default_binary_crossentropy; + } else if (input_layer->layer_type == ailayer_softmax_type){ + loss->crossentropy = aimath_f32_default_categorical_crossentropy; + } else { + // Error: Input layer type not supported + #ifdef AIDEBUG_PRINT_ERROR_MESSAGES + printf("\n!!! Error: Input layer type not supported\n"); + #endif // AIDEBUG_PRINT_ERROR_MESSAGES + return 0; + } + + return ailoss_crossentropy(loss, input_layer); +} diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailoss/ailoss_crossentropy_default.h b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailoss/ailoss_crossentropy_default.h new file mode 100644 index 0000000..67e6421 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailoss/ailoss_crossentropy_default.h @@ -0,0 +1,73 @@ +/** + * \file basic/default/ailoss/ailoss_crossentropy_default.h + * \internal + * \date 14.01.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link ailoss_crossentropy.h Cross-Entropy loss \endlink + * + * Hardware independent implementations of the Cross-Entropy loss in \link aimath_f32.h F32 \endlink data-type. + * For more information about the Cross-Entropy loss refer to ailoss_mse.h. + */ + +#ifndef AILOSS_CROSSENTROPY_DEFAULT +#define AILOSS_CROSSENTROPY_DEFAULT + +#include "basic/base/ailoss/ailoss_crossentropy.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct ailoss_crossentropy ailoss_crossentropy_f32_t; + +/** @brief Initializes and connect a \link ailoss_crossentropy.h Cross-Entropy loss \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * The labels must me either binary (when the output layer is a Sigmoid layer), for example + * @f[ + * \left( \begin{array}{ccc} 1 & 0 & 0 & 1 \\ 1 & 1 & 1 & 0 \\ 0 & 0 & 1 & 0 \end{array}\right) + * @f] + * + * or row wise one-hot encoded (when the output layer is a Softmax layer), for example + * @f[ + * \left( \begin{array}{ccc} 0 & 0 & 0 & 1 \\ 1 & 0 & 0 & 0 \\ 0 & 0 & 1 & 0 \end{array}\right) + * @f] + * + * If you want to provide labels as integers, please use ailoss_crossentropy_sparse8_f32_default() loss. + * + * Example: Create the loss structure:\n + * \code{.c} + * ailoss_crossentropy_f32_t crossentropy_loss; + * \endcode + * + * Example: Initialize and connect the loss to the layer structure:\n + * \code{.c} + * aimodel_t model; + * ... + * model.output_layer = ailayer_sigmoid_f32_default(&sigmoid_layer, x); + * + * model.loss = ailoss_crossentropy_f32_default(&crossentropy_loss, model.output_layer); + * \endcode + * + * @param *loss The loss structure to initialize. + * @param *input_layer The output layer of the model (**Must be either a Sigmoid or a Softmax layer!**). + * @return The (successfully) initialized loss structure. + */ +ailoss_t *ailoss_crossentropy_f32_default(ailoss_crossentropy_f32_t *loss, ailayer_t *input_layer); + +#endif // AILOSS_CROSSENTROPY_DEFAULT + diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailoss/ailoss_mse_default.c b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailoss/ailoss_mse_default.c new file mode 100644 index 0000000..7af3d78 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailoss/ailoss_mse_default.c @@ -0,0 +1,36 @@ +/** + * \file basic/default/ailoss/ailoss_mse_default.c + * \version 2.0alpha + * \date 28.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/ailoss/ailoss_mse_default.h" + + +ailoss_t *ailoss_mse_f32_default(ailoss_mse_f32_t *loss, ailayer_t *input_layer) +{ + loss->dtype = aif32; + + loss->tensor_sub = aimath_f32_default_tensor_sub; + loss->norm_squared = aimath_f32_default_norm_squared; + + return ailoss_mse(loss, input_layer); +} diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailoss/ailoss_mse_default.h b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailoss/ailoss_mse_default.h new file mode 100644 index 0000000..a294287 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/ailoss/ailoss_mse_default.h @@ -0,0 +1,58 @@ +/** + * \file basic/default/ailoss/ailoss_mse_default.h + * \internal + * \date 28.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link ailoss_mse.h Mean Squared Error loss \endlink + * + * Hardware independent implementations of the Mean Squared Error loss in \link aimath_f32.h F32 \endlink data-type. + * For more information about the Mean Squared Error loss refer to ailoss_mse.h. + */ + +#ifndef AILOSS_MSE_DEFAULT +#define AILOSS_MSE_DEFAULT + +#include "basic/base/ailoss/ailoss_mse.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct ailoss_mse ailoss_mse_f32_t; + +/** @brief Initializes and connect a \link ailoss_mse.h Mean Squared Error loss \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * Example: Create the loss structure:\n + * \code{.c} + * ailoss_mse_f32_t mse_loss; + * \endcode + * + * Example: Initialize and connect the loss to the layer structure:\n + * \code{.c} + * aimodel_t model; + * ... + * model.loss = ailoss_mse_f32_default(&mse_loss, model.output_layer); + * \endcode + * + * @param *loss The loss structure to initialize. + * @param *input_layer The output layer of the model. + * @return The (successfully) initialized loss structure. + */ +ailoss_t *ailoss_mse_f32_default(ailoss_mse_f32_t *loss, ailayer_t *input_layer); + +#endif // AILOSS_MSE_DEFAULT diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/aimath/aimath_f32_default.c b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/aimath/aimath_f32_default.c new file mode 100644 index 0000000..04c97d2 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/aimath/aimath_f32_default.c @@ -0,0 +1,551 @@ +/** + * \file basic/default/aimath/aimath_f32_default.c + * \version 2.0alpha + * \date 25.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/aimath/aimath_f32_default.h" +#include + + +void aimath_f32_default_linear(const aitensor_t *a, const aitensor_t *b, const aitensor_t *c, aitensor_t *result) +{ + uint16_t i, j, k; + float sum; + + float *a_data = (float *) a->data; + float *b_data = (float *) b->data; + float *c_data = c != 0 ? (float *) c->data : 0; + float *result_data = (float *) result->data; + +#ifdef SHAPE_CHECK + if(a->shape[1] != b->shape[0]) + { + LOG_E("MatMul input shapes doesn't match.\n"); + return; + } + if(a->shape[0] != result->shape[0] || b->shape[1] != result->shape[1]) + { + LOG_E("MatMul output shape doesn't match.\n"); + return; + } +#endif + + for(i = 0; i < a->shape[0]; i++) + { + for(j = 0; j < b->shape[1]; j++) + { + sum = 0.0f; + for(k = 0; k < a->shape[1]; k++) + { + sum += a_data[i*a->shape[1] + k] * b_data[k*b->shape[1] + j]; + } + if(c != 0){ + // Bias add + sum += c_data[j]; + } + result_data[i*b->shape[1] + j] = sum; + } + } + return; +} + +void aimath_f32_default_mat_mul(const aitensor_t *a, const aitensor_t *b, aitensor_t *result){ + aimath_f32_default_linear(a, b, 0, result); +} + +void aimath_f32_default_multiply(const aitensor_t *a, const aitensor_t *b, aitensor_t *result) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(a); i++) + { + ((float *) result->data)[i] = ((float *) a->data)[i] * ((float *) b->data)[i]; + } + return; +} + +void aimath_f32_default_divide(const aitensor_t *a, const aitensor_t *b, aitensor_t *result) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(a); i++) + { + ((float *) result->data)[i] = ((float *) a->data)[i] / ((float *) b->data)[i]; + } + return; +} + +void aimath_f32_default_scalar_mul(const void *scalar, const aitensor_t *a, aitensor_t *result) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(a); i++) + { + ((float *) result->data)[i] = *((float *) scalar) * ((float *) a->data)[i]; + } + return; +} + +void aimath_f32_default_scalar_add(const void *scalar, const aitensor_t *a, aitensor_t *result) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(a); i++) + { + ((float *) result->data)[i] = *((float *) scalar) + ((float *) a->data)[i]; + } + return; +} + +/** +* Add 2D tensors +*/ +void aimath_f32_default_tensor_add(const aitensor_t *a, const aitensor_t *b, aitensor_t *result) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(a); i++) + { + ((float *) result->data)[i] = ((float *) a->data)[i] + ((float *) b->data)[i]; + } + return; +} + +void aimath_f32_default_tensor_sub(const aitensor_t *a, const aitensor_t *b, aitensor_t *result) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(a); i++) + { + ((float *) result->data)[i] = ((float *) a->data)[i] - ((float *) b->data)[i]; + } + return; +} + +// only for 2D tensors +// a: f32 +// b: u8 +// result: f32 +void aimath_f32_default_tensor_sub_sparse8(const aitensor_t *a, const aitensor_t *b, aitensor_t *result) +{ + uint32_t i, index; + for(i = 0; i < a->shape[0]; i++) + { + index = a->shape[1] * i + ((uint8_t *) b->data)[i]; + ((float *) result->data)[index] = ((float *) a->data)[index] - 1.0f; + } + return; +} + +void aimath_f32_default_copy_tensor(const aitensor_t *from, aitensor_t *to) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(from); i++) + { + ((float *) to->data)[i] = ((float *) from->data)[i]; + } + return; +} + +void aimath_f32_default_transpose_vector(aitensor_t *vector) +{ + uint16_t temp; + temp = vector->shape[0]; + vector->shape[0] = vector->shape[1]; + vector->shape[1] = temp; + return; +} + +void aimath_f32_default_norm_squared(const aitensor_t *x, void *result) +{ + uint32_t i; + + *((float *) result) = 0.0f; + + for(i = 0; i < x->shape[0] * x->shape[1]; i++) + { + *((float *) result) += ((float *) x->data)[i] * ((float *) x->data)[i]; + } + return; +} + +void aimath_f32_default_sum(const aitensor_t *x, void *result) +{ + uint32_t i; + + *((float *) result) = 0.0f; + + for(i = 0; i < x->shape[0] * x->shape[1]; i++) + { + *((float *) result) += ((float *) x->data)[i]; + } + return; +} + +void aimath_f32_default_min(const aitensor_t *x, void *result) +{ + uint32_t i; + float min_value = FLT_MAX; + + for(i = 0; i < aimath_tensor_elements(x); i++) + { + if(((float *) x->data)[i] < min_value){ + min_value = ((float *) x->data)[i]; + } + } + *((float *) result) = min_value; + return; +} + + +void aimath_f32_default_max(const aitensor_t *x, void *result) +{ + uint32_t i; + float max_value = -FLT_MAX; + + for(i = 0; i < aimath_tensor_elements(x); i++) + { + if(((float *) x->data)[i] > max_value){ + max_value = ((float *) x->data)[i]; + } + } + *((float *) result) = max_value; + return; +} + + +void aimath_f32_default_sigmoid(const aitensor_t *x, aitensor_t *result) +{ + uint32_t i; + + for(i = 0; i < aimath_tensor_elements(x); i++) + { + ((float *) result->data)[i] = 1.0f / (1.0f + expf(- ((float *) x->data)[i])); + } + return; +} + +void aimath_f32_default_d_sigmoid(const aitensor_t *sigmoid_x, aitensor_t *result) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(sigmoid_x); i++) + { + // sigmoid'(x) = sigmoid(x) * (1 - sigmoid(x)) + ((float *) result->data)[i] = ((float *) sigmoid_x->data)[i] * (1.0f - ((float *) sigmoid_x->data)[i]); + } + return; +} + +void aimath_f32_default_tanh(const aitensor_t *x, aitensor_t *result) +{ + uint32_t i; + float temp; + for(i = 0; i < aimath_tensor_elements(x); i++) + { + temp = expf(((float *) x->data)[i]); + ((float *) result->data)[i] = (temp - (1.0f/temp)) / (temp + (1.0f/temp)); + } + return; +} + +void aimath_f32_default_d_tanh(const aitensor_t *tanh_x, aitensor_t *result) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(tanh_x); i++) + { + // tanh'(x) = 1 - (tanh(x))^2 + ((float *) result->data)[i] = 1.0f - (((float *) tanh_x->data)[i] * ((float *) tanh_x->data)[i]); + } + return; +} + +void aimath_f32_default_relu(const aitensor_t *x, aitensor_t *result) +{ + uint32_t i; + + for(i = 0; i < aimath_tensor_elements(x); i++) + { + ((float *) result->data)[i] = ((float *) x->data)[i] > 0.0f ? ((float *) x->data)[i] : 0.0f; + } + return; +} + +void aimath_f32_default_d_relu(const aitensor_t *x, aitensor_t *result) +{ + uint32_t i; + + for(i = 0; i < aimath_tensor_elements(x); i++) + { + ((float *) result->data)[i] = ((float *) x->data)[i] >= 0.0f ? 1.0f : 0.0f; + } + return; +} + +void aimath_f32_default_leaky_relu(const aitensor_t *x, const void *alpha, aitensor_t *result) +{ + uint32_t i; + + for(i = 0; i < aimath_tensor_elements(x); i++) + { + ((float *) result->data)[i] = ((float *) x->data)[i] >= 0.0f ? ((float *) x->data)[i] : ((float *) x->data)[i] * *((float *) alpha); + } + return; +} + +void aimath_f32_default_d_leaky_relu(const aitensor_t *x, const void *alpha, aitensor_t *result) +{ + uint32_t i; + + for(i = 0; i < aimath_tensor_elements(x); i++) + { + ((float *) result->data)[i] = ((float *) x->data)[i] >= 0.0f ? 1.0f : *((float *) alpha); + } + return; +} + +void aimath_f32_default_elu(const aitensor_t *x, const void *alpha, aitensor_t *result) +{ + uint32_t i; + + for(i = 0; i < aimath_tensor_elements(x); i++) + { + ((float *) result->data)[i] = ((float *) x->data)[i] > 0.0f ? ((float *) x->data)[i] : (*((float *) alpha) * (exp(((float *) x->data)[i]) - 1.0f)); + } + return; +} + +void aimath_f32_default_d_elu(const aitensor_t *x, const void *alpha, aitensor_t *result) +{ + uint32_t i; + + for(i = 0; i < aimath_tensor_elements(x); i++) + { + ((float *) result->data)[i] = ((float *) x->data)[i] > 0.0f ? 1.0f : (*((float *) alpha) * expf(((float *) x->data)[i])); + } + return; +} + +void aimath_f32_default_softmax(const aitensor_t *x, aitensor_t *result) +{ + uint32_t i, j; + float max; + float exp_sum; + + float *x_data = (float *) x->data; + float *result_data = (float *) result->data; + + // Multiplier for array index calculation + uint16_t multiplier = 1; + for(i = x->dim - 1; i >= 1; i--){ + multiplier *= x->shape[i]; + } + + // Do for every dataset. (0 is batch dimension) + for(i = 0; i < x->shape[0]; i++){ + // calc max value for numeric stability + max = x_data[0]; + for(j = 0; j < multiplier; j++) + { + if(x_data[i * multiplier + j] > max) max = x_data[i * multiplier + j]; + } + // calc exp functions + exp_sum = 0.0f; + for(j = 0; j < multiplier; j++) + { + result_data[i * multiplier + j] = aimath_f32_default_expf_fast(x_data[i * multiplier + j] - max); + exp_sum += result_data[i * multiplier + j]; + } + //calc softmax + for(j = 0; j < multiplier; j++) + { + result_data[i * multiplier + j] = result_data[i * multiplier + j] / exp_sum; + } + } + return; +} + +void aimath_f32_default_softsign(const aitensor_t *x, aitensor_t *result) +{ + uint32_t i; + + for(i = 0; i < aimath_tensor_elements(x); i++) + { + ((float *) result->data)[i] = ((float *) x->data)[i] / (1.0f + fabs(((float *) x->data)[i])); + } + return; +} + +void aimath_f32_default_d_softsign(const aitensor_t *x, aitensor_t *result) +{ + uint32_t i; + + for(i = 0; i < aimath_tensor_elements(x); i++) + { + ((float *) result->data)[i] = 1.0f / pow((1.0f + abs(((float *) x->data)[i])), 2); + } + return; +} + +// predicted data: f32 +// target_data: f32 +void aimath_f32_default_binary_crossentropy(const aitensor_t *predicted_data, const aitensor_t *target_data, void *result) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(predicted_data); i++) + { + if(((float *) target_data->data)[i] != 0){ + *((float *) result) -= ((float *) target_data->data)[i] * log(((float *) predicted_data->data)[i]) + + (1.0f - ((float *) target_data->data)[i]) * log(1.0f - ((float *) predicted_data->data)[i]); + } + } + return; +} + +// predicted data: f32 +// target_data: f32 +void aimath_f32_default_categorical_crossentropy(const aitensor_t *predicted_data, const aitensor_t *target_data, void *result) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(predicted_data); i++) + { + if(((float *) target_data->data)[i] != 0){ + *((float *) result) -= ((float *) target_data->data)[i] * log(((float *) predicted_data->data)[i]); + } + } + return; +} + +// 2D tensors only +// predicted data: f32 +// target_data: u8 +void aimath_f32_default_categorical_crossentropy_sparse8(const aitensor_t *predicted_data, const aitensor_t *target_data, void *result) +{ + uint32_t i, index; + for(i = 0; i < target_data->shape[0]; i++) + { + index = i * predicted_data->shape[1] + ((uint8_t *) target_data->data)[i]; + *((float *) result) -= log(((float *) predicted_data->data)[index]); + } + return; +} + +void aimath_f32_default_sqrt(const aitensor_t *x, aitensor_t *result) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(x); i++) + { + ((float *) result->data)[i] = sqrt(((float *) x->data)[i]); + } + return; +} + +void aimath_f32_default_zero_tensor(aitensor_t *tensor) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(tensor); i++) + { + ((float *) tensor->data)[i] = 0.0f; + } + return; +} + +void aimath_f32_default_init_zeros(aitensor_t *tensor) +{ + aimath_f32_default_zero_tensor(tensor); + return; +} + +void aimath_f32_default_tensor_init_uniform(aitensor_t *tensor, float from, float to) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(tensor); i++) + { + ((float *) tensor->data)[i] = ((float) rand() / (float) RAND_MAX) * (to - from) + from; + } + return; +} + +/* Glorot Uniform weight Initialization + * + * Glorot uniform initializer, also called Xavier uniform initializer. + * + * Glorot et al., 2010 + * + * \ref http://proceedings.mlr.press/v9/glorot10a/glorot10a.pdf + * \ref https://prateekvishnu.medium.com/xavier-and-he-normal-he-et-al-initialization-8e3d7a087528 + */ +void aimath_f32_default_init_glorot_uniform(aitensor_t *tensor) +{ + float fan_in, fan_out, fan_avg; + if(tensor->dim == 2) + { + fan_in = tensor->shape[0]; + fan_out = tensor->shape[1]; + } + else if(tensor->dim == 4) + { + fan_in = tensor->shape[1] * tensor->shape[2] * tensor->shape[3]; // In channel * kernel_elems + fan_out = tensor->shape[0] * tensor->shape[2] * tensor->shape[3]; // Out channel * kernel_elems + } + + fan_avg = (fan_in + fan_out) / 2.0f; + float r = sqrt(3.0f / fan_avg); + aimath_f32_default_tensor_init_uniform(tensor, -r, r); +} + +/* He Uniform weight Initialization + * + * He et al, 2015 + * + * \ref http://arxiv.org/abs/1502.01852 + * \ref https://prateekvishnu.medium.com/xavier-and-he-normal-he-et-al-initialization-8e3d7a087528 + */ +void aimath_f32_default_init_he_uniform(aitensor_t *tensor) +{ + float fan_in, fan_avg; + if(tensor->dim == 2) + { + fan_in = tensor->shape[0]; + } + else if(tensor->dim == 4) + { + fan_in = tensor->shape[1] * tensor->shape[2] * tensor->shape[3]; // In channel * kernel_elems + } + + fan_avg = fan_in / 2.0f; + float r = sqrt(3.0f / fan_avg); + aimath_f32_default_tensor_init_uniform(tensor, -r, r); +} + +//Info(?): http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.9.4508&rep=rep1&type=pdf +//ToDo: Mit dem in der Quelle erwhnten Parameter c rumspielen +float aimath_f32_default_expf_fast(float x) +{ + if(x > 80.0f){ + x = 80.0f; + } + else if(x < -80.0f){ + x = -80.0f; + } + + union { float f; int32_t x; } u; + //max-values for a ca. [-88 , 88] + // a = 2^23 / ln(2) ; b = 127 * 2^23 + // x_int32 = a * x_float + b + u.x = (int32_t) (12102203.0f * x + 1064631197.0f); + return u.f; + +} diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/aimath/aimath_f32_default.h b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/aimath/aimath_f32_default.h new file mode 100644 index 0000000..043209f --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/aimath/aimath_f32_default.h @@ -0,0 +1,1225 @@ +/** + * \file basic/default/aimath/aimath_f32_default.h + * \internal + * \date 25.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Math functions for \link aimath_f32.h F32 \endlink data type, default implementation + * + * These functions can be used when no hardware specific implementation is available. + */ + +#ifndef AIMATH_F32_DEFAULT +#define AIMATH_F32_DEFAULT + +#include +#include +#include +#include + +#include "basic/base/aimath/aimath_f32.h" + +/** @brief Performs a matrix multiplication of \link aimath_f32.h F32 \endlink matrices a and b and adds a vector c to each row + * + * The addition of the horizontal vector c is performed via broadcast, i.e. element wise in each column + * Mathematically this broadcast is equal to multiplying c with an vertical vector (with the same number of elements as c) + * and adding the result to a * b + * + * @f[ + * result = a \cdot b + \left( \begin{array}{c} + 1 \\ + 1 \\ + \vdots \\ + 1 \\ + \end{array}\right) \cdot c + * @f] + * + * Example: + * @f[ + * a = \left( \begin{array}{rrr} + 1 & 2 & 3 \\ + 4 & 5 & 6 \\ + 7 & 8 & 9 + \end{array}\right) + * @f] + * + * @f[ + * b = \left( \begin{array}{rr} + 1 & 0 \\ + 0 & 1 \\ + 0 & 0 + \end{array}\right) + * @f] + * + * @f[ + * c = \left( \begin{array}{rr} + 2 & 5 + \end{array}\right) + * @f] + * + * @f[ + * result = a \cdot b + \left( \begin{array}{r} 1 \\ + 1 \\ + 1 \\ + \end{array}\right) \cdot c + * @f] + * + * @f[ + * = \left( \begin{array}{rr} + 1 & 2 \\ + 4 & 5 \\ + 7 & 8 + \end{array}\right) + + \left( \begin{array}{rr} + 2 & 5 \\ + 2 & 5 \\ + 2 & 5 + \end{array}\right) + * @f] + * + * @details + * + * @f[ + * = \left( \begin{array}{rr} 3 & 7 \\ + 6 & 10 \\ + 9 & 13 + \end{array}\right) + * @f] + * + * Example: + * \code{.c} + * uint16_t a_shape[2] = {3, 3}; + * float a_data[3*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f, + * 7.0f, 8.0f, 9.0f}; + * aitensor_t a = AITENSOR_2D_F32(a_shape, a_data); + * + * uint16_t b_shape[2] = {3, 2}; + * float b_data[3*2] = {1.0f, 0.0f, + * 0.0f, 1.0f, + * 0.0f, 0.0f}; + * aitensor_t b = AITENSOR_2D_F32(b_shape, b_data); + * + * uint16_t c_shape[2] = {1, 2}; + * float c_data[1*2] = {2.0f, 5.0f}; + * aitensor_t c = AITENSOR_2D_F32(c_shape, c_data); + * + * uint16_t result_shape[2] = {3, 2}; + * float result_data[3*2]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_linear(&a, &b, &c, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *a Q31 matrix a (2D tensor of shape [N x K]) + * @param *b Q31 matrix b (2D tensor of shape [K x M]) + * @param *c Q31 vector c (2D tensor of shape [1 x M]) + * @param *result Resulting Q31 matrix (2D tensor of shape [N x M]) + */ +void aimath_f32_default_linear(const aitensor_t *a, const aitensor_t *b, const aitensor_t *c, aitensor_t *result); + +/** @brief Performs a matrix multiplication of \link aimath_f32.h F32 \endlink matrices a and b + * + * @f[ + * result = a \cdot b + * @f] + * + * Example: + * \code{.c} + * uint16_t a_shape[2] = {3, 3}; + * float a_data[3*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f, + * 7.0f, 8.0f, 9.0f}; + * aitensor_t a = AITENSOR_2D_F32(a_shape, a_data); + * + * uint16_t b_shape[2] = {3, 2}; + * float b_data[3*2] = {1.0f, 0.0f, + * 0.0f, 1.0f, + * 0.0f, 0.0f}; + * aitensor_t b = AITENSOR_2D_F32(b_shape, b_data); + * + * uint16_t result_shape[2] = {3, 2}; + * float result_data[3*2]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_mat_mul(&a, &b, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *a F32 matrix a (2D tensor of shape [N x K]) + * @param *b F32 matrix b (2D tensor of shape [K x M]) + * @param *result Resulting F32 matrix of the multiplication (2D tensor of shape [N x M]) + */ +void aimath_f32_default_mat_mul(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + +/** @brief Performs an element wise multiplication of \link aimath_f32.h F32 \endlink tensors a and b (Hadamard product) + * + * @f[ + * result = a \circ b + * @f] + * + * Example: + * \code{.c} + * uint16_t a_shape[2] = {2, 3}; + * float a_data[2*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f}; + * aitensor_t a = AITENSOR_2D_F32(a_shape, a_data); + * + * uint16_t b_shape[2] = {2, 3}; + * float b_data[2*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f}; + * aitensor_t b = AITENSOR_2D_F32(b_shape, b_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_multiply(&a, &b, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *a F32 tensor a (N-D tensor) + * @param *b F32 tensor b (N-D tensor) + * @param *result Resulting F32 tensor of the element wise multiplication (N-D tensor) + */ +void aimath_f32_default_multiply(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + +/** @brief Performs an element wise division of \link aimath_f32.h F32 \endlink tensors a and b (Hadamard division) + * + * @f[ + * result = a \oslash b + * @f] + * + * Example: + * \code{.c} + * uint16_t a_shape[2] = {2, 3}; + * float a_data[2*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f}; + * aitensor_t a = AITENSOR_2D_F32(a_shape, a_data); + * + * uint16_t b_shape[2] = {2, 3}; + * float b_data[2*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f}; + * aitensor_t b = AITENSOR_2D_F32(b_shape, b_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_divide(&a, &b, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *a F32 tensor a (N-D tensor) + * @param *b F32 tensor b (N-D tensor) + * @param *result Resulting F32 tensor of the element wise division (N-D tensor) + */ +void aimath_f32_default_divide(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + +/** @brief Performs a scalar multiplication (scaling) of \link aimath_f32.h F32 \endlink tensor a and a scalar + * + * @f[ + * result = scalar \cdot a + * @f] + * + * Example: + * \code{.c} + * uint16_t a_shape[2] = {2, 3}; + * float a_data[2*3] = {0.0f, 1.0f, 2.0f, + * 3.0f, 4.0f, 5.0f}; + * aitensor_t a = AITENSOR_2D_F32(a_shape, a_data); + * + * float scalar = 0.1f; + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_scalar_mul(&scalar, &a, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *scalar Scalar (type aiscalar_f32_t / float) + * @param *a F32 tensor a (N-D tensor) + * @param *result Resulting F32 tensor of the scalar multiplication (N-D tensor) + */ +void aimath_f32_default_scalar_mul(const void *scalar, const aitensor_t *a, aitensor_t *result); + +/** @brief Performs an element wise addition of a scalar to a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result = a + \left( \begin{array}{ccc} 1 & \ldots & 1 \\ + \vdots & \ddots & \vdots \\ + 1 & \ldots & 1 + \end{array}\right) \cdot scalar + * @f] + * + * Example: + * \code{.c} + * uint16_t a_shape[2] = {2, 3}; + * float a_data[2*3] = {0.0f, 1.0f, 2.0f, + * 3.0f, 4.0f, 5.0f}; + * aitensor_t a = AITENSOR_2D_F32(a_shape, a_data); + * + * float scalar = 0.1f; + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_scalar_add(&scalar, &a, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *scalar Scalar (type aiscalar_f32_t / float) + * @param *a F32 tensor a (N-D tensor) + * @param *result Resulting F32 tensor of the element wise scalar addition (N-D tensor) + */ +void aimath_f32_default_scalar_add(const void *scalar, const aitensor_t *a, aitensor_t *result); + +/** @brief Performs an element wise addition of \link aimath_f32.h F32 \endlink tensors a and b + * + * @f[ + * result = a + b + * @f] + * + * Example: + * \code{.c} + * uint16_t a_shape[2] = {2, 3}; + * float a_data[2*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f}; + * aitensor_t a = AITENSOR_2D_F32(a_shape, a_data); + * + * uint16_t b_shape[2] = {2, 3}; + * float b_data[2*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f}; + * aitensor_t b = AITENSOR_2D_F32(b_shape, b_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_tensor_add(&a, &b, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *a F32 tensor a (N-D tensor) + * @param *b F32 tensor b (N-D tensor) + * @param *result Resulting F32 tensor of the element wise addition (N-D tensor) + */ +void aimath_f32_default_tensor_add(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + +/** @brief Performs a element wise subtraction of \link aimath_f32.h F32 \endlink tensors a and b + * + * @f[ + * result = a - b + * @f] + * + * Example: + * \code{.c} + * uint16_t a_shape[2] = {2, 3}; + * float a_data[2*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f}; + * aitensor_t a = AITENSOR_2D_F32(a_shape, a_data); + * + * uint16_t b_shape[2] = {2, 3}; + * float b_data[2*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f}; + * aitensor_t b = AITENSOR_2D_F32(b_shape, b_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_tensor_sub(&a, &b, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *a F32 tensor a (N-D tensor) + * @param *b F32 tensor b (N-D tensor) + * @param *result Resulting F32 tensor of the element wise subtraction (N-D tensor) + */ +void aimath_f32_default_tensor_sub(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + +/** @brief Performs a subtraction between a \link aimath_f32.h F32 \endlink matrix a and a \link aimath_u8.h U8 \endlink sparse matrix b + * + * This function can subtract a row wise one-hot encoded matrix in sparse representation + * (just the integer index of the 1 is stored) from a normal F32 matrix a. + * + * For example the matrix + * @f[ + * \left( \begin{array}{ccc} 0 & 0 & 0 & 1 \\ 1 & 0 & 0 & 0 \\ 0 & 0 & 1 & 0 \end{array}\right) + * @f] + * in sparse representation is + * @f[ + * \left( \begin{array}{ccc} 3 \\ 0 \\ 2 \end{array}\right) + * @f] + * + * The result is then calculated as + * @f[ + * result_{ij} = \begin{cases} + a_{ij} - 1 & \text{if } j = b_i\\ + a_{ij} & \text{if } j \neq b_i + \end{cases} + * @f] + * + * Example: + * \code{.c} + * uint16_t a_shape[2] = {2, 3}; + * float a_data[2*3] = {0.2f, 0.1f, 0.7f, + * 0.9f, 0.1f, 0.0f}; + * aitensor_t a = AITENSOR_2D_F32(a_shape, a_data); + * + * uint16_t b_shape[2] = {2, 1}; + * uint8_t b_data[2*1] = {2, + * 0}; + * aitensor_t b = AITENSOR_2D_U8(b_shape, b_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_tensor_sub_sparse8(&a, &b, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *a F32 matrix a (2D tensor of shape [N x M]) + * @param *b U8 sparse matrix b (2D tensor of shape [N x 1]) + * @param *result Resulting F32 tensor of the subtraction (2D tensor of shape [N x M]) + */ +void aimath_f32_default_tensor_sub_sparse8(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + +/** @brief Performs an element wise copy of \link aimath_f32.h F32 \endlink tensors + * + * @f[ + * to \leftarrow from + * @f] + * + * Dimension and shape of from and to tensors have to be the same. + * + * Example: + * \code{.c} + * uint16_t from_shape[2] = {2, 3}; + * float from_data[2*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f}; + * aitensor_t from = AITENSOR_2D_F32(from_shape, from_data); + * + * uint16_t to_shape[2] = {2, 3}; + * float to_data[2*3]; + * aitensor_t to = AITENSOR_2D_F32(to_shape, to_data); + * + * aimath_f32_default_copy_tensor(&from, &to); + * + * print_aitensor(&to); + * \endcode + * + * @param *from F32 tensor to copy from (N-D tensor) + * @param *to F32 tensor to copy to (N-D tensor) + */ +void aimath_f32_default_copy_tensor(const aitensor_t *from, aitensor_t *to); + +/** @brief Transposes a \link aimath_f32.h F32 \endlink vector + * + * The given tensor must be a vector (2D tensor of shape [1 x N] or [N x 1]). + * + * @f[ + * vector \leftarrow vector^T + * @f] + * + * Example: + * \code{.c} + * uint16_t vector_shape[2] = {1, 3}; + * float vector_data[1*3] = {1.0f, 2.0f, 3.0f}; + * aitensor_t vector = AITENSOR_2D_F32(vector_shape, vector_data); + * + * aimath_f32_default_transpose_vector(&vector); + * + * print_aitensor(&vector); + * \endcode + * + * @param *vector F32 vector (2D tensor of shape [1 x N] or [N x 1]) + */ +void aimath_f32_default_transpose_vector(aitensor_t *vector); + +/** @brief Calculates the squared sum of all elements in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result = \sum_i x_{i}^2 + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = {0.0f, 1.0f, 2.0f, + * 3.0f, 4.0f, 5.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * float result; + * + * aimath_f32_default_norm_squared(&x, &result); + * + * print_aiscalar(&result, aif32); + * \endcode + * + * @param *x F32 tensor x (N-D tensor) + * @param *result Scalar result (type aiscalar_f32_t / float) + */ +void aimath_f32_default_norm_squared(const aitensor_t *x, void *result); + +/** @brief Calculates the sum of all elements in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result = \sum_i x_{i} + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = {0.0f, 1.0f, 2.0f, + * 3.0f, 4.0f, 5.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * float result; + * + * aimath_f32_default_sum(&x, &result); + * + * print_aiscalar(&result, aif32); + * \endcode + * + * @param *x F32 tensor x (N-D tensor) + * @param *result Scalar result (type aiscalar_f32_t / float) + */ +void aimath_f32_default_sum(const aitensor_t *x, void *result); + +/** @brief Identifies the minimum value in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result = min(x) + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = {0.0f, 1.0f, 2.0f, + * 3.0f, 4.0f, 5.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * float result; + * + * aimath_f32_default_min(&x, &result); + * + * print_aiscalar(&result, aif32); + * \endcode + * + * @param *x F32 tensor x to get the minimum value of (N-D tensor) + * @param *result Scalar result (type aiscalar_f32_t / float) + */ +void aimath_f32_default_min(const aitensor_t *x, void *result); + +/** @brief Identifies the maximum value in a \link aimath_f32.h F32 \endlink tensor + * + * @details + * + * @f[ + * result = max(x) + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = {0.0f, 1.0f, 2.0f, + * 3.0f, 4.0f, 5.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * float result; + * + * aimath_f32_default_max(&x, &result); + * + * print_aiscalar(&result, aif32); + * \endcode + * + * @param *x F32 tensor x to get the maximum value of (N-D tensor) + * @param *result Scalar result (type aiscalar_f32_t / float) + */ +void aimath_f32_default_max(const aitensor_t *x, void *result); + +/** @brief Calculates the sigmoid of each element in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{i} = \sigma(x_{i}) = \frac{1}{1 + e^{-x_{i}}} + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_sigmoid(&x, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *x F32 tensor to calculate the sigmoid from (N-D tensor) + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_sigmoid(const aitensor_t *x, aitensor_t *result); + +/** @brief Calculates the derivative sigmoid of each element in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{i} = \sigma'(x_{i}) = \sigma(x_{i}) \cdot (1 - \sigma(x_{i})) + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_sigmoid(&x, &result); + * aimath_f32_default_d_sigmoid(&result, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *sigmoid_x F32 tensor with the sigmoid values \f$ \sigma(x_{i}) \f$ (N-D tensor) + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_d_sigmoid(const aitensor_t *sigmoid_x, aitensor_t *result); + +/** @brief Calculates the tanh of each element in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{i} = \tanh(x_{i}) = \frac{e^{x_i} - e^{-x_i}}{e^{x_i} + e^{-x_i}} + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_tanh(&x, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *x F32 tensor to calculate the tanh from (N-D tensor) + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_tanh(const aitensor_t *x, aitensor_t *result); + +/** @brief Calculates the tanh derivative of each element in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{i} = tanh'(x_{i}) = 1 - tanh(x_{i})^2 + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_tanh(&x, &result); + * aimath_f32_default_d_tanh(&result, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *tanh_x F32 tensor with the tanh values \f$ \tanh(x_{i}) \f$ (N-D tensor) + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_d_tanh(const aitensor_t *tanh_x, aitensor_t *result); + +/** @brief Calculates the rectifier (ReLU) value of each element in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{i} = max(0, x_{i}) + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_relu(&x, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *x F32 tensor to calculate the ReLU from (N-D tensor) + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_relu(const aitensor_t *x, aitensor_t *result); + + +/** @brief Calculates the rectifier (ReLU) derivative of each element in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{ij} = \begin{cases} + 0 & \text{if } x_i < 0\\ + 1 & \text{if } x_i \geq 0 + \end{cases} + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_d_relu(&x, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *x F32 tensor to calculate the ReLU derivative from (N-D tensor) + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_d_relu(const aitensor_t *x, aitensor_t *result); + +/** @brief Calculates the leaky rectifier (leaky ReLU) value of each element in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{i} = \begin{cases} + \alpha \cdot x_i & \text{if } x_i < 0 \\ + x_i & \text{if } x_i \geq 0 + \end{cases} + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * float alpha = 0.01f; + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_leaky_relu(&x, &alpha, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *x F32 tensor to calculate the leaky ReLU from (N-D tensor) + * @param *alpha Scalar \f$ \alpha \f$ (type aiscalar_f32_t / float) for the leakage + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_leaky_relu(const aitensor_t *x, const void *alpha, aitensor_t *result); + +/** @brief Calculates the leaky rectifier (leaky ReLU) derivative of each element in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{i} = \begin{cases} + \alpha & \text{if } x_i < 0\\ + 1 & \text{if } x_i \geq 0 + \end{cases} + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * float alpha = 0.01f; + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_d_leaky_relu(&x, &alpha, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *x F32 tensor to calculate the leaky ReLU derivative from (N-D tensor) + * @param *alpha Scalar \f$ \alpha \f$ (type aiscalar_f32_t / float) for the leakage + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_d_leaky_relu(const aitensor_t *x, const void *alpha, aitensor_t *result); + +/** @brief Calculates the exponential rectifier (ELU) value of each element in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{i} = \begin{cases} + \alpha \cdot (e^{x_i} - 1) & \text{if } x_i < 0 \\ + x_i & \text{if } x_i \geq 0 + \end{cases} + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * float alpha = 1.0f; + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_elu(&x, &alpha, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *x F32 tensor to calculate the ELU from (N-D tensor) + * @param *alpha Scalar \f$ \alpha \f$ (type aiscalar_f32_t / float) + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_elu(const aitensor_t *x, const void *alpha, aitensor_t *result); + +/** @brief Calculates the exponential rectifier (ELU) derivative of each element in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{i} = \begin{cases} + \alpha \cdot e^{x_i} & \text{if } x_i < 0\\ + 1 & \text{if } x_i \geq 0 + \end{cases} + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * float alpha = 1.0f; + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_d_elu(&x, &alpha, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *x F32 tensor to calculate the ELU derivative from (N-D tensor) + * @param *alpha Scalar \f$ \alpha \f$ (type aiscalar_f32_t / float) + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_d_elu(const aitensor_t *x, const void *alpha, aitensor_t *result); + +/** @brief Calculates the softmax value of each row of a \link aimath_f32.h F32 \endlink matrix + * + * @f[ + * result_{i} = \frac{e^{x_i}}{\sum_{j=1}^{K} e^{x_j}} + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_softmax(&x, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *x F32 matrix to calculate the softmax from (2D tensor) + * @param *result Resulting F32 matrix (2D tensor) + */ +void aimath_f32_default_softmax(const aitensor_t *x, aitensor_t *result); + +/** @brief Calculates the softsign value of each element in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{i} = \frac {x_i} {1 + |x_i|} + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_softsign(&x, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *x F32 tensor to calculate the softsign from (N-D tensor) + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_softsign(const aitensor_t *x, aitensor_t *result); + +/** @brief Calculates the softsign derivative of each element in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{i} = \frac {x_i} {(1 + |x_i|)^2} + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_d_softsign(&x, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *x F32 tensor to calculate the softsign derivative from (N-D tensor) + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_d_softsign(const aitensor_t *x, aitensor_t *result); + +/** @brief Calculates the binary cross entropy between the \link aimath_f32.h F32 \endlink predicted and the target data + * + * @f[ + * result = - \sum_i (target_i \cdot \log(predicted_i) + (1 - target_i) \cdot \log(1 - predicted_i)) + * @f] + * + * Example: + * \code{.c} + * uint16_t p_shape[2] = {2, 3}; + * float p_data[2*3] = {0.8f, 0.1f, 0.7f, + * 0.2f, 0.3f, 0.0f}; + * aitensor_t p = AITENSOR_2D_F32(p_shape, p_data); + * + * uint16_t t_shape[2] = {2, 3}; + * float t_data[2*3] = {1.0f, 0.0f, 1.0f, + * 0.0f, 0.0f, 0.0f}; + * aitensor_t t = AITENSOR_2D_F32(t_shape, t_data); + * + * float result; + * + * aimath_f32_default_binary_crossentropy(&p, &t, &result); + * + * print_aiscalar(&result, aif32); + * \endcode + * + * @param *predicted_data F32 matrix with the predicted or calculated values (2D tensor) + * @param *target_data F32 matrix with the target data / true values / labels (2D tensor with binary values 0 or 1) + * @param *result Resulting F32 matrix (2D tensor) + */ +void aimath_f32_default_binary_crossentropy(const aitensor_t *predicted_data, const aitensor_t *target_data, void *result); + +/** @brief Calculates the categorical cross entropy between the \link aimath_f32.h F32 \endlink predicted and the target data + * + * @f[ + * result = - \sum_i target_i \cdot \log(predicted_i) + * @f] + * + * Example: + * \code{.c} + * uint16_t p_shape[2] = {2, 3}; + * float p_data[2*3] = {0.2f, 0.1f, 0.7f, + * 0.9f, 0.1f, 0.0f}; + * aitensor_t p = AITENSOR_2D_F32(p_shape, p_data); + * + * uint16_t t_shape[2] = {2, 3}; + * float t_data[2*3] = {0.0f, 0.0f, 1.0f, + * 1.0f, 0.0f, 0.0f}; + * aitensor_t t = AITENSOR_2D_F32(t_shape, t_data); + * + * float result; + * + * aimath_f32_default_categorical_crossentropy(&p, &t, &result); + * + * print_aiscalar(&result, aif32); + * \endcode + * + * @param *predicted_data F32 matrix with the predicted or calculated values (2D tensor) + * @param *target_data F32 matrix with the target data / true values / labels (2D tensor, rows one-hot encoded) + * @param *result Resulting F32 matrix (2D tensor) + */ +void aimath_f32_default_categorical_crossentropy(const aitensor_t *predicted_data, const aitensor_t *target_data, void *result); + +/** @brief Calculates the categorical Cross-Entropy between the \link aimath_f32.h F32 \endlink predicted data and the \link aimath_u8.h U8 \endlink target data in sparse representation + * + * This function can calculate the crossentropy between a row wise one-hot encoded matrix in sparse representation + * (just the integer index of the 1 is stored) and a normal F32 matrix a. + * + * For example the matrix + * @f[ + * \left( \begin{array}{ccc} 0 & 0 & 0 & 1 \\ 1 & 0 & 0 & 0 \\ 0 & 0 & 1 & 0 \end{array}\right) + * @f] + * in sparse representation is + * @f[ + * \left( \begin{array}{ccc} 3 \\ 0 \\ 2 \end{array}\right) + * @f] + * + * The result is then calculated from the one-hot encoded target matrix: + * @f[ + * result = - \sum_i target_{one-hot,i} \cdot \log(predicted_i) + * @f] + * + * Example: + * \code{.c} + * uint16_t p_shape[2] = {2, 3}; + * float p_data[2*3] = {0.2f, 0.1f, 0.7f, + * 0.9f, 0.1f, 0.0f}; + * aitensor_t p = AITENSOR_2D_F32(p_shape, p_data); + * + * uint16_t t_shape[2] = {2, 1}; + * uint8_t t_data[2*1] = {2, + * 0}; + * aitensor_t t = AITENSOR_2D_U8(t_shape, t_data); + * + * float result; + * + * aimath_f32_default_categorical_crossentropy_sparse8(&p, &t, &result); + * + * print_aiscalar(&result, aif32); + * \endcode + * + * @param *predicted_data F32 matrix with the predicted or calculated values (2D tensor of shape [N x M]) + * @param *target_data U8 matrix with the target data / true values / labels (2D tensor of shape [N x 1] with true class indices) + * @param *result Resulting F32 matrix (2D tensor of shape [N x M]) + */ +void aimath_f32_default_categorical_crossentropy_sparse8(const aitensor_t *predicted_data, const aitensor_t *target_data, void *result); + +/** @brief Calculates the element wise square root of a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{i} = \sqrt{x_{i}} + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_sqrt(&x, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *x F32 tensor to calculate the square root from (N-D tensor) + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_sqrt(const aitensor_t *x, aitensor_t *result); + +/** @brief Fills a \link aimath_f32.h F32 \endlink tensor with zeros + * + * @f[ + * tensor_{i} = 0 + * @f] + * + * Example: + * \code{.c} + * uint16_t tensor_shape[2] = {2, 3}; + * float tensor_data[2*3]; + * aitensor_t tensor = AITENSOR_2D_F32(tensor_shape, tensor_data); + * + * aimath_f32_default_zero_tensor(&tensor); + * + * print_aitensor(&tensor); + * \endcode + * + * In the F32 implementation of this function, there is no difference between aimath_f32_default_zero_tensor() + * and aimath_f32_default_init_zeros(). + * + * @param *tensor F32 tensor to set to zero (N-D tensor) + */ +void aimath_f32_default_zero_tensor(aitensor_t *tensor); + +/** @brief Fills a \link aimath_f32.h F32 \endlink tensor with zeros + * + * @f[ + * tensor_{i} = 0 + * @f] + * + * Example: + * \code{.c} + * uint16_t tensor_shape[2] = {2, 3}; + * float tensor_data[2*3]; + * aitensor_t tensor = AITENSOR_2D_F32(tensor_shape, tensor_data); + * + * aimath_f32_default_init_zeros(&tensor); + * + * print_aitensor(&tensor); + * \endcode + * + * In the F32 implementation of this function, there is no difference between aimath_f32_default_zero_tensor() + * and aimath_f32_default_init_zeros(). + * + * @param *tensor F32 tensor to set to zero (N-D tensor) + */ +void aimath_f32_default_init_zeros(aitensor_t *tensor); + +/** @brief Fills a \link aimath_f32.h F32 \endlink tensor with random numbers created from a uniform distribution within given range + * + * @f[ + * tensor_i \in \mathcal{U(from, to)} + * @f] + * + * Example: + * \code{.c} + * uint16_t tensor_shape[2] = {2, 3}; + * float tensor_data[2*3]; + * aitensor_t tensor = AITENSOR_2D_F32(tensor_shape, tensor_data); + * + * aimath_f32_default_tensor_init_uniform(&tensor, -1.5f, 1.5f); + * + * print_aitensor(&tensor); + * \endcode + * + * @param *tensor F32 tensor to initialize with random numbers (N-D tensor) + * @param from Minimum value of the uniform distribution + * @param to Maximum value of the uniform distribution + */ +void aimath_f32_default_tensor_init_uniform(aitensor_t *tensor, float from, float to); + +/** @brief Fills a \link aimath_f32.h F32 \endlink tensor with random numbers uniformly within given range, according to Glorot et al. + * + * @f[ + * fan_{avg} = \frac{fan_{in} + fan_{out}}{2} + * @f] + * @f[ + * r = \sqrt{\frac{3}{fan_{avg}}} + * @f] + * @f[ + * tensor_i \in \mathcal{U(-r, r)} + * @f] + * + * Example: + * \code{.c} + * uint16_t tensor_shape[2] = {2, 3}; + * float tensor_data[2*3]; + * aitensor_t tensor = AITENSOR_2D_F32(tensor_shape, tensor_data); + * + * aimath_f32_default_init_glorot_uniform(&tensor); + * + * print_aitensor(&tensor); + * \endcode + * + * @see Glorot et al., 2010 ( http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf ) + * + * @param *tensor F32 tensor to initialize with random numbers (N-D tensor) + */ +void aimath_f32_default_init_glorot_uniform(aitensor_t *tensor); + +/** @brief Fills a \link aimath_f32.h F32 \endlink tensor with uniformly drawn random numbers within given range, according to He et al. + * + * @f[ + * fan_{avg} = \frac{fan_{in}}{2} + * @f] + * @f[ + * r = \sqrt{\frac{3}{fan_{avg}}} + * @f] + * @f[ + * tensor_i \in \mathcal{U(-r, r)} + * @f] + * + * Example: + * \code{.c} + * uint16_t tensor_shape[2] = {2, 3}; + * float tensor_data[2*3]; + * aitensor_t tensor = AITENSOR_2D_F32(tensor_shape, tensor_data); + * + * aimath_f32_default_init_he_uniform(&tensor); + * + * print_aitensor(&tensor); + * \endcode + * + * @see He et al., 2015 ( https://www.cv-foundation.org/openaccess/content_iccv_2015/html/He_Delving_Deep_into_ICCV_2015_paper.html ) + * + * @param *tensor F32 tensor to initialize with random numbers (N-D tensor) + */ +void aimath_f32_default_init_he_uniform(aitensor_t *tensor); + +/** @brief Fast approximation of the exponential function + * + * @see http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.9.4508&rep=rep1&type=pdf + * + * @param x Input of the exponential function + */ +float aimath_f32_default_expf_fast(const float x); + +#endif // AIMATH_F32_DEFAULT + diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/aiopti/aiopti_adam_default.c b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/aiopti/aiopti_adam_default.c new file mode 100644 index 0000000..0f42a68 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/aiopti/aiopti_adam_default.c @@ -0,0 +1,88 @@ +/** + * \file basic/default/aiopti/aiopti_adam_default.c + * \version 2.0alpha + * \date 12.11.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/aiopti/aiopti_adam_default.h" + +aiopti_t *aiopti_adam_f32_default(aiopti_adam_f32_t *opti) +{ + opti->base.base.dtype = aif32; + + opti->base.base.learning_rate = &opti->learning_rate; + + opti->base.beta1 = &opti->beta1; + opti->base.beta2 = &opti->beta2; + opti->base.eps = &opti->eps; + + opti->base.beta1t = &opti->beta1t; + opti->base.beta2t = &opti->beta2t; + opti->base.one_minus_beta1 = &opti->one_minus_beta1; + opti->base.one_minus_beta2 = &opti->one_minus_beta2; + opti->base.lrt = &opti->lrt; + + // ToDo: Check Initialization + opti->beta1t = opti->beta1; + opti->beta2t = opti->beta2; + opti->one_minus_beta1 = 1.0f - opti->beta1; + opti->one_minus_beta2 = 1.0f - opti->beta2; + //opti->lrt = *((float *) opti->base.base.learning_rate) * sqrtf(1.0f - opti->beta2) / (1.0f - opti->beta1); + + // Set f32 function of optimizer base function begin_step() and end_step() + opti->base.base.begin_step = aiopti_adam_f32_default_begin_step; + opti->base.base.end_step = aiopti_adam_f32_default_end_step; + + // Set f32 math functions of adam optimizer + opti->base.multiply = aimath_f32_default_multiply; + opti->base.divide = aimath_f32_default_divide; + opti->base.tensor_add = aimath_f32_default_tensor_add; + opti->base.tensor_sub = aimath_f32_default_tensor_sub; + opti->base.scalar_mul = aimath_f32_default_scalar_mul; + opti->base.scalar_add = aimath_f32_default_scalar_add; + opti->base.sqrt = aimath_f32_default_sqrt; + + opti->base.zero_tensor = aimath_f32_default_zero_tensor; + + return aiopti_adam(&opti->base); +} + +void aiopti_adam_f32_default_begin_step(aiopti_t *self) +{ + aiopti_adam_t *opti = (aiopti_adam_t *)(self->optimizer_configuration); + + // lr_t = lr * sqrt(1-beta2^t) / (1-beta1^t) + *((float *) opti->lrt) = *((float *) self->learning_rate) * sqrtf(1.0f - *((float *) opti->beta2t)) / (1.0f - *((float *) opti->beta1t)); + return; +} + +void aiopti_adam_f32_default_end_step(aiopti_t *self) +{ + aiopti_adam_t *opti = (aiopti_adam_t *)(self->optimizer_configuration); + + // beta^t <= beta^t * beta; + *((float *) opti->beta1t) = *((float *) opti->beta1t) * *((float *) opti->beta1); + *((float *) opti->beta2t) = *((float *) opti->beta2t) * *((float *) opti->beta2); + *((float *) opti->one_minus_beta1) = 1.0f - *((float *) opti->beta1); + *((float *) opti->one_minus_beta2) = 1.0f - *((float *) opti->beta2); + + return; +} diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/aiopti/aiopti_adam_default.h b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/aiopti/aiopti_adam_default.h new file mode 100644 index 0000000..8103a3a --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/aiopti/aiopti_adam_default.h @@ -0,0 +1,137 @@ +/** + * \file basic/default/aiopti/aiopti_adam_default.h + * \internal + * \date 28.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link aiopti_adam.h Adam optimizer \endlink + * + * Hardware independent implementations of the Adam optimizer in \link aimath_f32.h F32 \endlink data-type. + * For more information about the Adam optimizer refer to aiopti_adam.h. + */ + +#ifndef AIOPTI_ADAM_DEFAULT +#define AIOPTI_ADAM_DEFAULT + +#include "basic/base/aiopti/aiopti_adam.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct aiopti_adam_f32 aiopti_adam_f32_t; + +/** @brief Data-type specific \link aiopti_adam.h Adam optimizer \endlink struct for \link aimath_f32.h F32 \endlink + * + * Adds data fields for the learning rate and the configuration values in \link aimath_f32.h F32 \endlink to the base implementation. + */ +struct aiopti_adam_f32 { + aiopti_adam_t base; /**< Inherited field members from general optimizer struct. */ + + /** @name Optimizer configuration + * @brief Required configuration parameters for the optimizer + * + * These fields have to be configured by the user before calling the initializer function. + */ + ///@{ + aiscalar_f32_t learning_rate; /**< Storage for aiopti.learning_rate scalar in F32 */ + + aiscalar_f32_t beta1; /**< Storage for aiopti_adam.beta1 scalar in F32 */ + aiscalar_f32_t beta2; /**< Storage for aiopti_adam.beta2 scalar in F32 */ + aiscalar_f32_t eps; /**< Storage for aiopti_adam.eps scalar in F32 */ + ///@} + + /** @name Variables for internal computation + * + * These fields are automatically configured in the initializer function. + */ + ///@{ + aiscalar_f32_t beta1t; /**< Storage for aiopti_adam.beta1t scalar in F32 */ + aiscalar_f32_t beta2t; /**< Storage for aiopti_adam.beta2t scalar in F32 */ + aiscalar_f32_t one_minus_beta1; /**< Storage for aiopti_adam.one_minus_beta1 scalar in F32 */ + aiscalar_f32_t one_minus_beta2; /**< Storage for aiopti_adam.one_minus_beta2 scalar in F32 */ + aiscalar_f32_t lrt; /**< Storage for aiopti_adam.lrt scalar in F32 */ + ///@} +}; + + +/** @brief Initializes an \link aiopti_adam.h Adam optimizer \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * Example: Create the optimizer structure:\n + * \code{.c} + * aiopti_adam_f32_t adam_optimizer = { + * .learning_rate = 0.01f, + * + * .beta1 = 0.9f, + * .beta2 = 0.999f, + * .eps = 1e-7f + * }; + * \endcode + * + * Example: Initialize the optimizer:\n + * \code{.c} + * aiopti_t *optimizer; + * + * optimizer = aiopti_adam_f32_default(&adam_optimizer); + * \endcode + * + * @param *opti The optimizer structure to initialize. + * @return The (successfully) initialized optimizer structure. + */ +aiopti_t *aiopti_adam_f32_default(aiopti_adam_f32_t *opti); + +/** @brief \link aimath_f32.h F32 \endlink default implementation of the aiopti.begin_step function for ADAM + * + * *Implementation of aiopti.begin_step.* + * + * The ADAM optimizer needs some modification of the learning rate in every optimization step. + * This function deals with aiscalars and has to be implemented for every data-type individually. + * + * The calculations are:\n + * @f[ + * lr_t \leftarrow lr \cdot \frac{\sqrt{1 - \beta^t_2}} {(1 - \beta_1)^t} + * @f] + * + * This is not primary time critical function, because it only deals with scalars -> No special hardware implementation necessary (but possible). + * + * @param *self The optimizer structure + */ +void aiopti_adam_f32_default_begin_step(aiopti_t *self); + +/** @brief \link aimath_f32.h F32 \endlink default implementation of the aiopti.end_step function for ADAM + * + * *Implementation of aiopti.end_step.* + * + * The ADAM optimizer needs some modification of the learning rate in every optimization step. + * This function deals with aiscalars and has to be implemented for every data-type individually. + * + * The calculations are:\n + * @f[ + * \beta^t_1 \leftarrow \beta^t_1 \cdot \beta_1 + * @f] + * + * @f[ + * \beta^t_2 \leftarrow \beta^t_2 \cdot \beta_2 + * @f] + * + * This is not primary time critical function, because it only deals with scalars -> No special hardware implementation necessary (but possible). + * + * @param *self The optimizer structure + */ +void aiopti_adam_f32_default_end_step(aiopti_t *self); + +#endif // AIOPTI_ADAM_DEFAULT diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/aiopti/aiopti_sgd_default.c b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/aiopti/aiopti_sgd_default.c new file mode 100644 index 0000000..03ee4e3 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/aiopti/aiopti_sgd_default.c @@ -0,0 +1,58 @@ +/** + * \file basic/default/aiopti/aiopti_sgd_default.c + * \version 2.0alpha + * \date 28.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/aiopti/aiopti_sgd_default.h" + +aiopti_t *aiopti_sgd_f32_default(aiopti_sgd_f32_t *opti) +{ + aiopti_t* return_opti; + + opti->base.base.dtype = aif32; + + // Call "constructor" of base "class" + return_opti = aiopti_sgd(&opti->base); + + return_opti->learning_rate = &(opti->learning_rate); + opti->base.momentum = &opti->momentum; + + // Check if a momentum is set and configure the right functions + if (opti->momentum != 0.0f) { + return_opti->update_params = aiopti_sgd_update_params_with_momentum; + return_opti->sizeof_optimem = aiopti_sgd_sizeof_optimem_with_momentum; + return_opti->init_optimem = aiopti_sgd_init_optimem_with_momentum; + } + else { + return_opti->update_params = aiopti_sgd_update_params_without_momentum; + return_opti->sizeof_optimem = aiopti_sgd_sizeof_optimem_without_momentum; + return_opti->init_optimem = aiopti_sgd_init_optimem_without_momentum; + } + + // Set f32 math functions of sgd optimizer + opti->base.zero_tensor = aimath_f32_default_zero_tensor; + opti->base.tensor_add = aimath_f32_default_tensor_add; + opti->base.tensor_sub = aimath_f32_default_tensor_sub; + opti->base.scalar_mul = aimath_f32_default_scalar_mul; + + return return_opti; +} diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/aiopti/aiopti_sgd_default.h b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/aiopti/aiopti_sgd_default.h new file mode 100644 index 0000000..94793dd --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/basic/default/aiopti/aiopti_sgd_default.h @@ -0,0 +1,75 @@ +/** + * \file basic/default/aiopti/aiopti_sgd_default.h + * \internal + * \date 28.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link aiopti_sgd.h Stochastic Gradient Descend optimizer \endlink + * + * Hardware independent implementations of the Stochastic Gradient Descend optimizer in \link aimath_f32.h F32 \endlink data-type. + * For more information about the Stochastic Gradient Descend optimizer refer to aiopti_sgd.h. + */ + +#ifndef AIOPTI_SGD_DEFAULT +#define AIOPTI_SGD_DEFAULT + +#include "basic/base/aiopti/aiopti_sgd.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct aiopti_sgd_f32 aiopti_sgd_f32_t; /**< New data type name for code reduction. */ + +/** @brief Data-type specific \link aiopti_sgd.h SGD optimizer \endlink struct for \link aimath_f32.h F32 \endlink + * + * Adds data fields for the learning rate and the momentum in \link aimath_f32.h F32 \endlink to the base implementation. + */ +struct aiopti_sgd_f32 { + aiopti_sgd_t base; /**< Inherited field members from general aiopti_sgd struct. */ + + aiscalar_f32_t learning_rate; /**< Storage for aiopti.learning_rate scalar in F32 */ + + aiscalar_f32_t momentum; /**< Storage for aiopti_sgd.momentum scalar in F32 */ +}; + +/** @brief Initializes a \link aiopti_sgd.h SGD optimizer \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * You can set the momentum to zero to save memory space. + * + * Example: Create the optimizer structure:\n + * \code{.c} + * aiopti_sgd_f32_t sgd_optimizer = { + * .learning_rate = 0.01f, + * + * .momentum = 0.9f + * }; + * \endcode + * + * Example: Initialize the optimizer:\n + * \code{.c} + * aiopti_t *optimizer; + * + * optimizer = aiopti_sgd_f32_default(&sgd_optimizer); + * \endcode + * + * @param *opti The optimizer structure to initialize. + * @return The (successfully) initialized optimizer structure. + */ +aiopti_t *aiopti_sgd_f32_default(aiopti_sgd_f32_t *opti); + +#endif // AIOPTI_SGD_DEFAULT diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/core/aifes_core.h b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/core/aifes_core.h new file mode 100644 index 0000000..202005f --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/core/aifes_core.h @@ -0,0 +1,466 @@ +/** + * \file core/aifes_core.h + * \internal + * \date 20.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief AIfES 2 core interface + * + * This interface defines the functional modules of AIfES 2 like model, layer, loss and optimizer. + * + * The structure of an aimodel and its components can look like in the following example for a simple multilayer perceptron - MLP: + * \image html functional_concept.jpg width=800px + */ + +#ifndef AIFES_CORE +#define AIFES_CORE + +#include "aifes_math.h" + +#define TRUE 1 +#define FALSE 0 + +// ToDo: Make enum from the values +#define AILAYER_RESULT_LOWER_BOUND 0 +#define AILAYER_RESULT_UPPER_BOUND 1 +#define AILAYER_DELTAS_LOWER_BOUND 2 +#define AILAYER_DELTAS_UPPER_BOUND 3 + +typedef struct ailayer ailayer_t; +typedef struct ailoss ailoss_t; +typedef struct aimodel aimodel_t; +typedef struct aiopti aiopti_t; + +typedef struct aicore_layertype aicore_layertype_t; +typedef struct aicore_losstype aicore_losstype_t; +typedef struct aicore_optitype aicore_optitype_t; + + +/** @brief Type indicator of the layer + * + * Every layer should have a constant global variable of this type which indicates the layer type. + * + * Example for the dense layer:\n + * \code{.c} + * const aicore_layertype_t ailayer_dense_type_s = { + * #ifdef AIDEBUG_PRINT_MODULE_SPECS + * .name = "Dense", + * .print_specs = ailayer_dense_print_specs + * #else + * .name = 0, + * .print_specs = 0 + * #endif + * }; + * const aicore_layertype_t *ailayer_dense_type = &ailayer_dense_type_s; + * \endcode + * + * Checks for the layer type can look like this:\n + * \code{.c} + * ailayer_t *example_layer; + * if(example_layer->layer_type == ailayer_dense_type){ + * ... + * } + * \endcode + */ +struct aicore_layertype { + const char *name; /**< Name of the layer type (for example "Dense") */ + + /** @brief Set a function to print specs of the layer (for example size, constants) + * + * This function should only be set in the debug mode when prints are required (to save memory otherwise) + * + * @param self The layer + * @param *print A function for printing (for example printf) + */ + void (*print_specs)(const ailayer_t *self, int (*print)(const char *format, ...)); +}; + +/** @brief Type indicator of the loss to check for the loss type + * + * Every loss should have a constant global variable of this type which indicates the loss type.\n + * Example for the mse loss:\n + * + * \code{.c} + * const aicore_losstype_t ailoss_mse_type_s = { + * #ifdef AIDEBUG_PRINT_MODULE_SPECS + * .name = "Mean Squared Error", + * .print_specs = ailoss_mse_print_spec + * #else + * .name = 0, + * .print_specs = 0 + * #endif + * }; + * const aicore_losstype_t *ailoss_mse_type = &ailoss_mse_type_s; + * \endcode + * + * Checks for the loss type can look like this:\n + * \code{.c} + * ailoss_t *example_loss; + * if(example_loss->loss_type == ailoss_mse_type){ + * ... + * } + * \endcode + */ +struct aicore_losstype { + const char *name; /**< Name of the loss type (for example "Mean Squared Error") */ + + /** @brief Set a function to print specs of the loss + * + * This function should only be set in the debug mode when prints are required (to save memory otherwise) + * + * @param self The loss + * @param *print A function for printing (for example printf) + */ + void (*print_specs)(const ailoss_t *self, int (*print)(const char *format, ...)); +}; + +/** @brief Type indicator of the optimizer to check for the optimizer type + * + * Every optimizer should have a constant global variable of this type which indicates the optimizer type.\n + * Example for the adam optimizer:\n + * + * \code{.c} + * const aicore_optitype_t aiopti_adam_type_s = { + * #ifdef AIDEBUG_PRINT_MODULE_SPECS + * .name = "ADAM", + * .print_specs = aiopti_adam_print_specs + * #else + * .name = 0, + * .print_specs = 0 + * #endif + * }; + * const aicore_optitype_t *aiopti_adam_type = &aiopti_adam_type_s; + * \endcode + * + * Checks for the optimizer type can look like this:\n + * \code{.c} + * aiopti_t *example_optimizer; + * if(example_optimizer->optimizer_type == aiopti_adam_type){ + * ... + * } + * \endcode + */ +struct aicore_optitype { + const char *name; /**< Name of the optimizer type (for example "ADAM") */ + + /** @brief Set a function to print specs of the optimizer + * + * This function should only be set in the debug mode when prints are required (to save memory otherwise) + * + * @param self The optimizer + * @param *print A function for printing (for example printf) + */ + void (*print_specs)(const aiopti_t *self, int (*print)(const char *format, ...)); +}; + +/** @brief AIfES artificial neural network model +* +* \image html aimodel.png width=500px +* +* This structure holds the model structure and compute graph of the ANN. (Refer to aifes_core.h for a structural overview)\n +* A aimodel contains several layers that are connected in a list or graph structure.\n +* When the model should be trained, a loss function is required. Otherwise it can be set to NULL. +*/ +struct aimodel { + ailayer_t *input_layer; /**< Input layer of the model that gets the input data. */ + ailayer_t *output_layer; /**< Output layer of the model. The output of a forwad pass is available in the result tensor of this layer. */ + + uint16_t layer_count; /**< Total number of layers of the model (usually autogenerated). */ + uint16_t trainable_params_count; /**< Total number of trainable parameter tensors */ + + ailoss_t *loss; /**< The loss or cost function of the model (only for training). */ +}; + + +/** @brief AIfES layer interface +* +* \image html ailayer.png width=200px +* +* The interface contains the necessary functions and parameters for inference and training on the model. +* (Refer to aifes_core.h for a structural overview) +* +* The call order of the functions for inference:\n +* \code{.c} +* +* for each layer in the model +* calc_result_shape() +* endfor +* +* for each layer in the model +* forward() +* endfor +* // The result of the inference is now in output_layer.result tensor +* \endcode +* +* The call order of the functions for training:\n +* \code{.c} +* +* for each layer in the model +* calc_result_shape() +* endfor +* +* // If the parameters are not already pretrained, a new parameter memory block can be created +* for each layer in the model +* allocate memory of size sizeof_paramem() +* set_paramem() +* endfor +* init_trainable_parameters_model() // Do some weight initialization +* +* for each layer in the model +* allocate memory of size sizeof_trainmem() +* set_trainmem() +* endfor +* +* init_trainmem_model() // For example set the optimem to zero +* for iterations +* for each batch in dataset +* zero_gradients_model() +* +* for each sample in the batch +* for each layer in the model +* forward() +* endfor +* +* calc_delta() +* for each layer in the model (reverse) +* backward() +* endfor +* endfor +* +* update_params_model() +* endfor +* endfor +* \endcode +*/ +struct ailayer { + const aicore_layertype_t *layer_type; /**< Type of the layer (for example ailayer_dense_type) */ + void *layer_configuration; /**< Layer specific configurations (back-link from abstract layer class to implementation) */ + + /** @name Layer connections + * Defines the model graph. + */ + ///@{ + ailayer_t *input_layer; + //ailayer_t *brother_input_layer; /**< (NOT_IN_USE) Chained list if multiple input layer are present else NULL. */ + + ailayer_t *output_layer; + //ailayer_t *brother_output_layer; /**< (NOT_IN_USE) Chained list if multiple output layer are present else NULL. */ + ///@} + + /** @name Inference and training scheduling order (Not in use yet) + * @brief The scheduler can execute the layers along this path. + */ + ///@{ + ailayer_t *next_scheduled; + ailayer_t *prev_scheduled; + ///@} + + aitensor_t result; /**< The result of the forward function is stored here */ + + /** @brief Result and delta min and max values. + * + * Returns the min/max values if the result can only be in certain value ranges (like sigmoid). + * This may be used e.g. for quantization algorithms. Set to NULL if not in use. + * + * @param self The layer + * @param selector Select the requested bound (AILAYER_RESULT_LOWER_BOUND, AILAYER_RESULT_UPPER_BOUND, AILAYER_DELTA_LOWER_BOUND, AILAYER_DELTA_UPPER_BOUND) + * @param result_bound A scalar of aiscalar type in which the bound value can be written + * @return TRUE if bound available else FALSE + */ + uint8_t (*get_result_bound)(const ailayer_t *self, const uint8_t selector, void *result_bound); + + /** @brief Calculate and write the shape to the result tensor. + * + * Made for easy creation of the model (layers can be connected to each other without worrying about the shapes). + * + * @param self The layer + */ + void (*calc_result_shape)(ailayer_t *self); + + /** @brief Calculate the forward pass and write the result to the result tensor. + * + * @param self The layer + */ + void (*forward)(ailayer_t *self); + + // Maybe for later purpose + //void (*sizeof_infmem)(struct aifes_layer_t *, void *); + //void (*set_infmem)(struct aifes_layer_t *, void *); + + // ------------------ Used for training only: ----------------------- + + aitensor_t deltas; /**< The result of the backward function is stored here. */ + + /** @name Training memory API + * @brief Makes the memory of the trainable params, the gradients and optimizer stuff accessible. + * + * This is used, for example, for the optimiser or debugging purposes. + */ + ///@{ + uint8_t trainable_params_count; /**< Number of trainable parameter tensors. */ + aitensor_t **trainable_params; /**< Array of tensor pointers with length trainable_params_count. */ + aitensor_t **gradients; /**< Array of tensor pointers with length trainable_params_count. */ + void **optimem; /**< Array of memory pointers with length trainable_params_count. */ + ///@} + + /** @brief Calculate the backward pass and write the result to the deltas tensor. + * + * @param self The layer + */ + void (*backward)(ailayer_t *self); + + /** @name Parameter memory + * @brief Calculate the size and set the memory for the parameter. + * + * This memory (for example for weights, bias, ...) will last through all the lifetime of the model.\n + * This is only intended for training when no initial weights are available. If the parameters are already known, + * set the parameter directly to the layer. + */ + ///@{ + uint32_t (*sizeof_paramem)(const ailayer_t *self); /**< Size of required memory (in bytes). */ + void (*set_paramem)(ailayer_t *self, void* memory_ptr); /**< Set and distribute the memory block internally. */ + ///@} + + /** @name Training memory + * @brief Calculate the size and set the working memory for the training + * + * This memory (for example for gradients, momentums, ...) is needed during the whole training process. + * If the training is finished, it can be deleted. + */ + ///@{ + uint32_t (*sizeof_trainmem)(const ailayer_t *self); /**< Size of required memory (in bytes). */ + void (*set_trainmem)(ailayer_t *self, void* memory_ptr); /**< Set and distribute the memory block internally. */ + ///@} +}; + + +/** @brief AIfES loss interface. +* +* \image html ailoss.png width=300px +* +* The interface contains the necessary functions and parameters for loss and error calculation on the target data. +* (Refer to aifes_core.h for a structural overview) +*/ +struct ailoss { + const aicore_losstype_t *loss_type; /**< Type of the loss (for example ailoss_mse_type) */ + void *loss_configuration; /**< Loss specific configurations (back-link from abstract loss class to implementation) */ + + ailayer_t connection_layer; /**< Dummy layer for docking to the layer structure */ + + /** @brief Calculate the loss / cost for the model on the given targets. + * + * @param self The layer + * @param target_data Tensor containing the target data / labels + * @param result Scalar in which the output can be written (aiscalar of same type as layer type). + */ + void (*calc_loss)(ailoss_t *self, const aitensor_t *target_data, void *result); + + /** @brief Calculate the error on the target data and write it to the deltas tensor of connection layer + * + * @param self The layer + * @param target_data Tensor containing the target data / labels + */ + void (*calc_delta)(ailoss_t *self, const aitensor_t *target_data); +}; + + +/** @brief AIfES optimizer interface. +* +* \image html aiopti.png width=300px +* +* The interface contains the necessary functions and parameters for parameter optimizers in backpropagation training. +* (Refer to aifes_core.h for a structural overview)\n +* Optimizers are responsible for updating the trainable parameters of the model with the gradients +* calculated in the backward pass. +* +* The call order of the functions is:\n +* \code{.c} +* allocate memory of size sizeof_optimem() +* init_optimem() +* +* for batch in dataset +* for each trainable parameter tensor in the model +* zero_gradients() +* endfor +* +* forward_model(batch) +* backward_model(batch) +* +* begin_step() +* for each trainable parameter tensor in the model +* update_params() +* endfor +* end_step() +* endfor +* \endcode +*/ +struct aiopti { + const aicore_optitype_t *optimizer_type; /**< Type of the optimizer (for example aiopti_sgd_type) */ + void *optimizer_configuration; /**< Optimizer specific configurations (back-link from abstract aiopti class to implementation) */ + const aimath_dtype_t *dtype; /**< The data-type of the parameter that the optimizer can optimize and the learning rate */ + + void *learning_rate; /**< The learning rate configures the training speed. The learning rate is an aiscalar_t value of given dtype. */ + + /** @brief Calculates the optimization memory size for a trainable parameter tensor. + * + * @param self The layer + * @param params The trainable parameter tensor + */ + uint32_t (*sizeof_optimem)(aiopti_t *self, const aitensor_t *params); + + /** @brief Initialize the optimization memory for a trainable parameter tensor. + * + * @param self The layer + * @param params The trainable parameter tensor + * @param gradients The associated gradients tensor + * @param optimem The associated optimization memory to initialize + */ + void (*init_optimem)(aiopti_t *self, const aitensor_t *params, const aitensor_t *gradients, void *optimem); + + /** @brief Set the gradient tensor to zero. + * + * @param self The layer + * @param params The gradient tensor + */ + void (*zero_gradients)(aiopti_t *self, aitensor_t *gradients); + + /** @brief Called in the beginning of every model optimization step for parameter initialization. + * + * @param self The layer + */ + void (*begin_step)(aiopti_t *self); + + /** @brief Performs an optimization step on the given tensor. + * + * @param self The layer + * @param params The trainable parameter tensor + * @param gradients The associated gradients tensor + * @param optimem The associated optimization memory to initialize + */ + void (*update_params)(aiopti_t *self, aitensor_t *params, const aitensor_t *gradients, void *optimem); + + /** @brief Called in the end of every model optimization step + * + * @param self The layer + */ + void (*end_step)(aiopti_t *self); +}; + + +#endif // AIFES_CORE diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/core/aifes_math.h b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/core/aifes_math.h new file mode 100644 index 0000000..9580f47 --- /dev/null +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/core/aifes_math.h @@ -0,0 +1,107 @@ +/** + * \file core/aifes_math.h + * \internal + * \date 20.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief AIfES 2 math interface + * + * This interface describes the basic structures to perform data type independent math in AIfES. +*/ + +#ifndef AIFES_MATH +#define AIFES_MATH + +#include +#include +#include +#include + + +#define SHAPE_CHECK /** +#include +#include +#include + +#include "aifes.h" + +#include "MNIST_test_data.h" +#include "MNIST_test_data_label.h" +#include "MNIST_weights.h" + +#define INPUTS 784 +#define NEURONS_1 10 +#define NEURONS_2 10 +#define OUTPUTS 10 +#define NUM_TEST_DATA 10000 + + +void predict_model_acc(float *model_acc, int num_data, int num_output_neurons, float *data_label, float *model_pred){ + /* + Calculates the accuracy of the neural network + */ + + float argmax_target_acc; + uint8_t argmax_target_index; + float argmax_pred_acc; + uint8_t argmax_pred_index; + *model_acc = 0; + + for(int i=0;i argmax_target_acc) { + argmax_target_acc = *((data_label+i*num_output_neurons)+j); + argmax_target_index = j; + } + if(*((model_pred+i*num_output_neurons)+j) > argmax_pred_acc) { + argmax_pred_acc = *((model_pred+i*num_output_neurons)+j); + argmax_pred_index = j; + } + } + if(argmax_target_index == argmax_pred_index){ + (*model_acc)++; + } + } + *model_acc = (*model_acc/num_data)*100; + +} + + +int main(int argc, char *argv[]) +{ + // loop variables + uint32_t i; + uint32_t j; + uint32_t k; + + float training_acc; + float test_acc; + + + uint16_t input_shape[] = {NUM_TEST_DATA, INPUTS}; // Definition of the input shape + aitensor_t input_tensor; // Creation of the input AIfES tensor + input_tensor.dtype = aif32; // Definition of the used data type, here float with 32 bits, different ones are available + input_tensor.dim = 2; // Dimensions of the tensor, here 2 dimensions, as specified at input_shape + input_tensor.shape = input_shape; // Set the shape of the input_tensor + input_tensor.data = MNIST_test_data; // Assign the input_data array to the tensor. It expects a pointer to the array where the data is stored + + + uint16_t target_shape[] = {NUM_TEST_DATA, OUTPUTS}; // Definition of the output shape + aitensor_t target_tensor; // Creation of the input AIfES tensor + target_tensor.dtype = aif32; // Definition of the used data type, here float with 32 bits, different ones are available + target_tensor.dim = 2; // Dimensions of the tensor, here 2 dimensions, as specified at input_shape + target_tensor.shape = target_shape; // Set the shape of the input_tensor + target_tensor.data = MNIST_test_data_label; // Assign the target_data array to the tensor. It expects a pointer to the array where the data is stored + + // Tensor for the output data (result after training). + // Same configuration as for the target tensor + float output_data[NUM_TEST_DATA][OUTPUTS]; + uint16_t output_shape[] = {NUM_TEST_DATA, OUTPUTS}; + aitensor_t output_tensor; + output_tensor.dtype = aif32; + output_tensor.dim = 2; + output_tensor.shape = output_shape; + output_tensor.data = output_data; + + // ---------------------------------- Layer definition --------------------------------------- + + // Input layer + uint16_t input_layer_shape[] = {1, INPUTS}; // Definition of the input layer shape (Must fit to the input tensor) + ailayer_input_t input_layer; // Creation of the AIfES input layer + input_layer.input_dim = 2; // Definition of the input dimension (Must fit to the input tensor) + input_layer.input_shape = input_layer_shape; // Handover of the input layer shape + + // Dense layer (hidden layer_1) + ailayer_dense_t dense_layer_1; // Creation of the AIfES hidden dense layer + dense_layer_1.neurons = NEURONS_1; // Number of neurons + dense_layer_1.weights.data = weights_data_dense_1; // Passing the hidden layer weights + dense_layer_1.bias.data = bias_data_dense_1; // Passing the hidden layer bias weights + + ailayer_relu_t relu_layer_1; // Relu activation function + + // Dense layer (hidden layer_2) + ailayer_dense_t dense_layer_2; // Creation of the AIfES hidden dense layer + dense_layer_2.neurons = NEURONS_2; // Number of neurons + dense_layer_2.weights.data = weights_data_dense_2; // Passing the hidden layer weights + dense_layer_2.bias.data = bias_data_dense_2; // Passing the hidden layer bias weights + + ailayer_relu_t relu_layer_2; // Relu activation function + + // Output dense layer + ailayer_dense_t dense_layer_3; // Creation of the AIfES ouput dense layer + dense_layer_3.neurons = OUTPUTS; // Number of neurons + dense_layer_3.weights.data = weights_data_dense_3; // Passing the output layer weights + dense_layer_3.bias.data = bias_data_dense_3; // Passing the output layer bias weights + + ailayer_softmax_t softmax_layer_3; // Softmax activation function + + + // --------------------------- Define the structure of the model ---------------------------- + + aimodel_t model; // AIfES model + ailayer_t *x; // Layer object from AIfES, contains the layers + + // Passing the layers to the AIfES model + model.input_layer = ailayer_input_f32_default(&input_layer); + x = ailayer_dense_f32_default(&dense_layer_1, model.input_layer); + x = ailayer_relu_f32_default(&relu_layer_1, x); + x = ailayer_dense_f32_default(&dense_layer_2, x); + x = ailayer_relu_f32_default(&relu_layer_2, x); + x = ailayer_dense_f32_default(&dense_layer_3, x); + x = ailayer_softmax_f32_default(&softmax_layer_3, x); + model.output_layer = x; + + aialgo_compile_model(&model); // Compile the AIfES model + + // -------------------------------- Allocate and schedule the working memory for training --------- + + uint32_t memory_size = aialgo_sizeof_inference_memory(&model); + printf("Required memory for the training (Intermediate results, gradients, optimization memory): %d Byte\n", memory_size); + + void *memory_ptr = malloc(memory_size); + + // Schedule the memory over the model + aialgo_schedule_inference_memory(&model, memory_ptr, memory_size); + + + // --------------------------------- Result with imported keras weights ---------------------------------- + + aialgo_inference_model(&model, &input_tensor, &output_tensor); + printf("\n"); + printf("Results with imported weights from keras:\n"); + + for (i = 0; i < 10; i++) { + for(k = 0; k < 2; k++) { + if(k==0){ + printf("real output %d:\n",i); + } + else { + printf("calculated output %d:\n",i); + } + for(j = 0; j < 10; j++) { + if(k==0){ + printf("%f",MNIST_test_data_label[i][j]); + printf("\t"); + } + else { + printf("%f",output_data[i][j]); + printf("\t"); + } + } + } + printf("\n\n"); + } + + predict_model_acc(&training_acc, NUM_TEST_DATA, OUTPUTS, &MNIST_test_data_label, &output_data); + printf("Model accuracy with imported keras weights: %.3f%%\n\n", training_acc); + + + free(memory_ptr); + + system("pause"); + + return 0; +} diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/.gitignore b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/.gitignore new file mode 100644 index 0000000..909ce84 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/.gitignore @@ -0,0 +1,7 @@ +bin/ +obj/ +/src/MNIST_training_data.h +/src/MNIST_training_data_label.h +/src/MNIST_test_data.h +/src/MNIST_test_data_label.h +/src/MNIST_weights.h \ No newline at end of file diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/LICENSE b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/LICENSE new file mode 100644 index 0000000..f288702 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/MNIST_retrain_keras.cbp b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/MNIST_retrain_keras.cbp new file mode 100644 index 0000000..6227fc9 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/MNIST_retrain_keras.cbp @@ -0,0 +1,186 @@ + + + + + + diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/MNIST_retrain_keras.depend b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/MNIST_retrain_keras.depend new file mode 100644 index 0000000..b2f1e69 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/MNIST_retrain_keras.depend @@ -0,0 +1,938 @@ +# depslib dependency file v1.0 +1631882463 source:c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\base\aialgo\aialgo_sequential_inference.c + "basic/base/aialgo/aialgo_sequential_inference.h" + "basic/default/aimath/aimath_f32_default.h" + + + + +1628682756 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\base\aialgo\aialgo_sequential_inference.h + "core/aifes_core.h" + "core/aifes_math.h" + "basic/base/aimath/aimath_basic.h" + "basic/default/ailayer/ailayer_dense_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\core\aifes_core.h + "aifes_math.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\core\aifes_math.h + + + + + +1628682756 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\base\aimath\aimath_basic.h + "core/aifes_math.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\default\ailayer\ailayer_dense_default.h + "basic/base/ailayer/ailayer_dense.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_dense.h + "core/aifes_core.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\default\aimath\aimath_f32_default.h + + + + + "basic/base/aimath/aimath_f32.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\base\aimath\aimath_f32.h + "core/aifes_math.h" + "basic/base/aimath/aimath_basic.h" + +1632130252 source:c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\base\aialgo\aialgo_sequential_training.c + "basic/base/aialgo/aialgo_sequential_training.h" + "basic/base/aialgo/aialgo_sequential_inference.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\base\aialgo\aialgo_sequential_training.h + "core/aifes_core.h" + "core/aifes_math.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_dense.c + "basic/base/ailayer/ailayer_dense.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_elu.c + "basic/base/ailayer/ailayer_elu.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_elu.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_input.c + "basic/base/ailayer/ailayer_input.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_input.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_leaky_relu.c + "basic/base/ailayer/ailayer_leaky_relu.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_leaky_relu.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_relu.c + "basic/base/ailayer/ailayer_relu.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_relu.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_sigmoid.c + "basic/base/ailayer/ailayer_sigmoid.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_sigmoid.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_softmax.c + "basic/base/ailayer/ailayer_softmax.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_softmax.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_softsign.c + "basic/base/ailayer/ailayer_softsign.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_softsign.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_tanh.c + "basic/base/ailayer/ailayer_tanh.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_tanh.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_template.c + "basic/base/ailayer/ailayer_template.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\base\ailayer\ailayer_template.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\base\ailoss\ailoss_crossentropy.c + "basic/base/ailoss/ailoss_crossentropy.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\base\ailoss\ailoss_crossentropy.h + "core/aifes_core.h" + "basic/base/ailayer/ailayer_softmax.h" + "basic/base/ailayer/ailayer_sigmoid.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\base\ailoss\ailoss_mse.c + "basic/base/ailoss/ailoss_mse.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\base\ailoss\ailoss_mse.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\base\aimath\aimath_basic.c + "basic/base/aimath/aimath_basic.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\base\aimath\aimath_f32.c + "basic/base/aimath/aimath_f32.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\base\aiopti\aiopti_adam.c + "basic/base/aiopti/aiopti_adam.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\base\aiopti\aiopti_adam.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\base\aiopti\aiopti_sgd.c + "basic/base/aiopti/aiopti_sgd.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\base\aiopti\aiopti_sgd.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\cmsis\ailayer\ailayer_dense_cmsis.c + "basic/default/ailayer/ailayer_dense_default.h" + "../../../aifes.h" + "basic/cmsis/ailayer/ailayer_dense_cmsis.h" + "basic/cmsis/aimath/aimath_f32_cmsis.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\aifes.h + "core/aifes_math.h" + "core/aifes_core.h" + "basic/base/aimath/aimath_f32.h" + "basic/base/aimath/aimath_basic.h" + "basic/base/ailayer/ailayer_dense.h" + "basic/base/ailayer/ailayer_input.h" + "basic/base/ailayer/ailayer_relu.h" + "basic/base/ailayer/ailayer_leaky_relu.h" + "basic/base/ailayer/ailayer_elu.h" + "basic/base/ailayer/ailayer_sigmoid.h" + "basic/base/ailayer/ailayer_tanh.h" + "basic/base/ailayer/ailayer_softmax.h" + "basic/base/ailayer/ailayer_softsign.h" + "basic/base/ailoss/ailoss_mse.h" + "basic/base/ailoss/ailoss_crossentropy.h" + "basic/base/aiopti/aiopti_sgd.h" + "basic/base/aiopti/aiopti_adam.h" + "basic/default/aimath/aimath_f32_default.h" + "basic/default/ailayer/ailayer_dense_default.h" + "basic/default/ailayer/ailayer_input_default.h" + "basic/default/ailayer/ailayer_relu_default.h" + "basic/default/ailayer/ailayer_leaky_relu_default.h" + "basic/default/ailayer/ailayer_elu_default.h" + "basic/default/ailayer/ailayer_sigmoid_default.h" + "basic/default/ailayer/ailayer_tanh_default.h" + "basic/default/ailayer/ailayer_softmax_default.h" + "basic/default/ailayer/ailayer_softsign_default.h" + "basic/default/ailoss/ailoss_mse_default.h" + "basic/default/ailoss/ailoss_crossentropy_default.h" + "basic/default/aiopti/aiopti_sgd_default.h" + "basic/default/aiopti/aiopti_adam_default.h" + "basic/cmsis/aimath/aimath_f32_cmsis.h" + "basic/cmsis/ailayer/ailayer_dense_cmsis.h" + "basic/base/aialgo/aialgo_sequential_inference.h" + "basic/base/aialgo/aialgo_sequential_training.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\default\ailayer\ailayer_input_default.h + "basic/base/ailayer/ailayer_input.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\default\ailayer\ailayer_relu_default.h + "basic/base/ailayer/ailayer_relu.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\default\ailayer\ailayer_leaky_relu_default.h + "basic/base/ailayer/ailayer_leaky_relu.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\default\ailayer\ailayer_elu_default.h + "basic/base/ailayer/ailayer_elu.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\default\ailayer\ailayer_sigmoid_default.h + "basic/base/ailayer/ailayer_sigmoid.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\default\ailayer\ailayer_tanh_default.h + "basic/base/ailayer/ailayer_tanh.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\default\ailayer\ailayer_softmax_default.h + "basic/base/ailayer/ailayer_softmax.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\default\ailayer\ailayer_softsign_default.h + "basic/base/ailayer/ailayer_softsign.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\default\ailoss\ailoss_mse_default.h + "basic/base/ailoss/ailoss_mse.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\default\ailoss\ailoss_crossentropy_default.h + "basic/base/ailoss/ailoss_crossentropy.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\default\aiopti\aiopti_sgd_default.h + "basic/base/aiopti/aiopti_sgd.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\default\aiopti\aiopti_adam_default.h + "basic/base/aiopti/aiopti_adam.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\cmsis\aimath\aimath_f32_cmsis.h + "basic/base/aimath/aimath_f32.h" + "../../../aifes.h" + +1628682756 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\cmsis\ailayer\ailayer_dense_cmsis.h + "../../../aifes.h" + "basic/base/ailayer/ailayer_dense.h" + "basic/cmsis/aimath/aimath_f32_cmsis.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\cmsis\aimath\aimath_f32_cmsis.c + "basic/cmsis/aimath/aimath_f32_cmsis.h" + "../../../aifes.h" + "arm_math.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\default\ailayer\ailayer_dense_default.c + "basic/default/ailayer/ailayer_dense_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\default\ailayer\ailayer_elu_default.c + "basic/default/ailayer/ailayer_elu_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\default\ailayer\ailayer_input_default.c + "basic/default/ailayer/ailayer_input_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\default\ailayer\ailayer_leaky_relu_default.c + "basic/default/ailayer/ailayer_leaky_relu_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\default\ailayer\ailayer_relu_default.c + "basic/default/ailayer/ailayer_relu_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\default\ailayer\ailayer_sigmoid_default.c + "basic/default/ailayer/ailayer_sigmoid_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\default\ailayer\ailayer_softmax_default.c + "basic/default/ailayer/ailayer_softmax_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\default\ailayer\ailayer_softsign_default.c + "basic/default/ailayer/ailayer_softsign_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\default\ailayer\ailayer_tanh_default.c + "basic/default/ailayer/ailayer_tanh_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\default\ailoss\ailoss_crossentropy_default.c + "basic/default/ailoss/ailoss_crossentropy_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\default\ailoss\ailoss_mse_default.c + "basic/default/ailoss/ailoss_mse_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\default\aimath\aimath_f32_default.c + "basic/default/aimath/aimath_f32_default.h" + + +1628682756 source:c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\default\aiopti\aiopti_adam_default.c + "basic/default/aiopti/aiopti_adam_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\basic\default\aiopti\aiopti_sgd_default.c + "basic/default/aiopti/aiopti_sgd_default.h" + +1632296400 source:c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\main.c + + + + + "aifes.h" + "MNIST_training_data.h" + "MNIST_training_data_label.h" + "MNIST_test_data.h" + "MNIST_test_data_label.h" + "MNIST_weights.h" + +1632137043 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\mnist_training_data.h + +1632137043 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\mnist_training_data_label.h + +1632233468 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\mnist_test_data.h + +1632233324 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\mnist_test_data_label.h + +1632296883 c:\users\dk100\documents\aifes\mnist_retrain_keras\aifes_for_arduino\src\mnist_weights.h + +1631882463 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\base\aialgo\aialgo_sequential_inference.c + "basic/base/aialgo/aialgo_sequential_inference.h" + "basic/default/aimath/aimath_f32_default.h" + + + + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\base\aialgo\aialgo_sequential_inference.h + "core/aifes_core.h" + "core/aifes_math.h" + "basic/base/aimath/aimath_basic.h" + "basic/default/ailayer/ailayer_dense_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\core\aifes_core.h + "aifes_math.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\core\aifes_math.h + + + + + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\base\aimath\aimath_basic.h + "core/aifes_math.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\default\ailayer\ailayer_dense_default.h + "basic/base/ailayer/ailayer_dense.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\base\ailayer\ailayer_dense.h + "core/aifes_core.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\default\aimath\aimath_f32_default.h + + + + + "basic/base/aimath/aimath_f32.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\base\aimath\aimath_f32.h + "core/aifes_math.h" + "basic/base/aimath/aimath_basic.h" + +1632130252 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\base\aialgo\aialgo_sequential_training.c + "basic/base/aialgo/aialgo_sequential_training.h" + "basic/base/aialgo/aialgo_sequential_inference.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\base\aialgo\aialgo_sequential_training.h + "core/aifes_core.h" + "core/aifes_math.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\base\ailayer\ailayer_dense.c + "basic/base/ailayer/ailayer_dense.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\base\ailayer\ailayer_elu.c + "basic/base/ailayer/ailayer_elu.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\base\ailayer\ailayer_elu.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\base\ailayer\ailayer_input.c + "basic/base/ailayer/ailayer_input.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\base\ailayer\ailayer_input.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\base\ailayer\ailayer_leaky_relu.c + "basic/base/ailayer/ailayer_leaky_relu.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\base\ailayer\ailayer_leaky_relu.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\base\ailayer\ailayer_relu.c + "basic/base/ailayer/ailayer_relu.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\base\ailayer\ailayer_relu.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\base\ailayer\ailayer_sigmoid.c + "basic/base/ailayer/ailayer_sigmoid.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\base\ailayer\ailayer_sigmoid.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\base\ailayer\ailayer_softmax.c + "basic/base/ailayer/ailayer_softmax.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\base\ailayer\ailayer_softmax.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\base\ailayer\ailayer_softsign.c + "basic/base/ailayer/ailayer_softsign.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\base\ailayer\ailayer_softsign.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\base\ailayer\ailayer_tanh.c + "basic/base/ailayer/ailayer_tanh.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\base\ailayer\ailayer_tanh.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\base\ailayer\ailayer_template.c + "basic/base/ailayer/ailayer_template.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\base\ailayer\ailayer_template.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\base\ailoss\ailoss_crossentropy.c + "basic/base/ailoss/ailoss_crossentropy.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\base\ailoss\ailoss_crossentropy.h + "core/aifes_core.h" + "basic/base/ailayer/ailayer_softmax.h" + "basic/base/ailayer/ailayer_sigmoid.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\base\ailoss\ailoss_mse.c + "basic/base/ailoss/ailoss_mse.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\base\ailoss\ailoss_mse.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\base\aimath\aimath_basic.c + "basic/base/aimath/aimath_basic.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\base\aimath\aimath_f32.c + "basic/base/aimath/aimath_f32.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\base\aiopti\aiopti_adam.c + "basic/base/aiopti/aiopti_adam.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\base\aiopti\aiopti_adam.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\base\aiopti\aiopti_sgd.c + "basic/base/aiopti/aiopti_sgd.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\base\aiopti\aiopti_sgd.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\cmsis\ailayer\ailayer_dense_cmsis.c + "basic/default/ailayer/ailayer_dense_default.h" + "../../../aifes.h" + "basic/cmsis/ailayer/ailayer_dense_cmsis.h" + "basic/cmsis/aimath/aimath_f32_cmsis.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\aifes.h + "core/aifes_math.h" + "core/aifes_core.h" + "basic/base/aimath/aimath_f32.h" + "basic/base/aimath/aimath_basic.h" + "basic/base/ailayer/ailayer_dense.h" + "basic/base/ailayer/ailayer_input.h" + "basic/base/ailayer/ailayer_relu.h" + "basic/base/ailayer/ailayer_leaky_relu.h" + "basic/base/ailayer/ailayer_elu.h" + "basic/base/ailayer/ailayer_sigmoid.h" + "basic/base/ailayer/ailayer_tanh.h" + "basic/base/ailayer/ailayer_softmax.h" + "basic/base/ailayer/ailayer_softsign.h" + "basic/base/ailoss/ailoss_mse.h" + "basic/base/ailoss/ailoss_crossentropy.h" + "basic/base/aiopti/aiopti_sgd.h" + "basic/base/aiopti/aiopti_adam.h" + "basic/default/aimath/aimath_f32_default.h" + "basic/default/ailayer/ailayer_dense_default.h" + "basic/default/ailayer/ailayer_input_default.h" + "basic/default/ailayer/ailayer_relu_default.h" + "basic/default/ailayer/ailayer_leaky_relu_default.h" + "basic/default/ailayer/ailayer_elu_default.h" + "basic/default/ailayer/ailayer_sigmoid_default.h" + "basic/default/ailayer/ailayer_tanh_default.h" + "basic/default/ailayer/ailayer_softmax_default.h" + "basic/default/ailayer/ailayer_softsign_default.h" + "basic/default/ailoss/ailoss_mse_default.h" + "basic/default/ailoss/ailoss_crossentropy_default.h" + "basic/default/aiopti/aiopti_sgd_default.h" + "basic/default/aiopti/aiopti_adam_default.h" + "basic/cmsis/aimath/aimath_f32_cmsis.h" + "basic/cmsis/ailayer/ailayer_dense_cmsis.h" + "basic/base/aialgo/aialgo_sequential_inference.h" + "basic/base/aialgo/aialgo_sequential_training.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\default\ailayer\ailayer_input_default.h + "basic/base/ailayer/ailayer_input.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\default\ailayer\ailayer_relu_default.h + "basic/base/ailayer/ailayer_relu.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\default\ailayer\ailayer_leaky_relu_default.h + "basic/base/ailayer/ailayer_leaky_relu.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\default\ailayer\ailayer_elu_default.h + "basic/base/ailayer/ailayer_elu.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\default\ailayer\ailayer_sigmoid_default.h + "basic/base/ailayer/ailayer_sigmoid.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\default\ailayer\ailayer_tanh_default.h + "basic/base/ailayer/ailayer_tanh.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\default\ailayer\ailayer_softmax_default.h + "basic/base/ailayer/ailayer_softmax.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\default\ailayer\ailayer_softsign_default.h + "basic/base/ailayer/ailayer_softsign.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\default\ailoss\ailoss_mse_default.h + "basic/base/ailoss/ailoss_mse.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\default\ailoss\ailoss_crossentropy_default.h + "basic/base/ailoss/ailoss_crossentropy.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\default\aiopti\aiopti_sgd_default.h + "basic/base/aiopti/aiopti_sgd.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\default\aiopti\aiopti_adam_default.h + "basic/base/aiopti/aiopti_adam.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\cmsis\aimath\aimath_f32_cmsis.h + "basic/base/aimath/aimath_f32.h" + "../../../aifes.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\cmsis\ailayer\ailayer_dense_cmsis.h + "../../../aifes.h" + "basic/base/ailayer/ailayer_dense.h" + "basic/cmsis/aimath/aimath_f32_cmsis.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\cmsis\aimath\aimath_f32_cmsis.c + "basic/cmsis/aimath/aimath_f32_cmsis.h" + "../../../aifes.h" + "arm_math.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\default\ailayer\ailayer_dense_default.c + "basic/default/ailayer/ailayer_dense_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\default\ailayer\ailayer_elu_default.c + "basic/default/ailayer/ailayer_elu_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\default\ailayer\ailayer_input_default.c + "basic/default/ailayer/ailayer_input_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\default\ailayer\ailayer_leaky_relu_default.c + "basic/default/ailayer/ailayer_leaky_relu_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\default\ailayer\ailayer_relu_default.c + "basic/default/ailayer/ailayer_relu_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\default\ailayer\ailayer_sigmoid_default.c + "basic/default/ailayer/ailayer_sigmoid_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\default\ailayer\ailayer_softmax_default.c + "basic/default/ailayer/ailayer_softmax_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\default\ailayer\ailayer_softsign_default.c + "basic/default/ailayer/ailayer_softsign_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\default\ailayer\ailayer_tanh_default.c + "basic/default/ailayer/ailayer_tanh_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\default\ailoss\ailoss_crossentropy_default.c + "basic/default/ailoss/ailoss_crossentropy_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\default\ailoss\ailoss_mse_default.c + "basic/default/ailoss/ailoss_mse_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\default\aimath\aimath_f32_default.c + "basic/default/aimath/aimath_f32_default.h" + + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\default\aiopti\aiopti_adam_default.c + "basic/default/aiopti/aiopti_adam_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\mnist_retrain_keras\src\basic\default\aiopti\aiopti_sgd_default.c + "basic/default/aiopti/aiopti_sgd_default.h" + +1631882463 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\base\aialgo\aialgo_sequential_inference.c + "basic/base/aialgo/aialgo_sequential_inference.h" + "basic/default/aimath/aimath_f32_default.h" + + + + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\base\aialgo\aialgo_sequential_inference.h + "core/aifes_core.h" + "core/aifes_math.h" + "basic/base/aimath/aimath_basic.h" + "basic/default/ailayer/ailayer_dense_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\core\aifes_core.h + "aifes_math.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\core\aifes_math.h + + + + + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\base\aimath\aimath_basic.h + "core/aifes_math.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\default\ailayer\ailayer_dense_default.h + "basic/base/ailayer/ailayer_dense.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\base\ailayer\ailayer_dense.h + "core/aifes_core.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\default\aimath\aimath_f32_default.h + + + + + "basic/base/aimath/aimath_f32.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\base\aimath\aimath_f32.h + "core/aifes_math.h" + "basic/base/aimath/aimath_basic.h" + +1632130252 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\base\aialgo\aialgo_sequential_training.c + "basic/base/aialgo/aialgo_sequential_training.h" + "basic/base/aialgo/aialgo_sequential_inference.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\base\aialgo\aialgo_sequential_training.h + "core/aifes_core.h" + "core/aifes_math.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\base\ailayer\ailayer_dense.c + "basic/base/ailayer/ailayer_dense.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\base\ailayer\ailayer_elu.c + "basic/base/ailayer/ailayer_elu.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\base\ailayer\ailayer_elu.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\base\ailayer\ailayer_input.c + "basic/base/ailayer/ailayer_input.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\base\ailayer\ailayer_input.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\base\ailayer\ailayer_leaky_relu.c + "basic/base/ailayer/ailayer_leaky_relu.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\base\ailayer\ailayer_leaky_relu.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\base\ailayer\ailayer_relu.c + "basic/base/ailayer/ailayer_relu.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\base\ailayer\ailayer_relu.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\base\ailayer\ailayer_sigmoid.c + "basic/base/ailayer/ailayer_sigmoid.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\base\ailayer\ailayer_sigmoid.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\base\ailayer\ailayer_softmax.c + "basic/base/ailayer/ailayer_softmax.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\base\ailayer\ailayer_softmax.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\base\ailayer\ailayer_softsign.c + "basic/base/ailayer/ailayer_softsign.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\base\ailayer\ailayer_softsign.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\base\ailayer\ailayer_tanh.c + "basic/base/ailayer/ailayer_tanh.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\base\ailayer\ailayer_tanh.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\base\ailayer\ailayer_template.c + "basic/base/ailayer/ailayer_template.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\base\ailayer\ailayer_template.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\base\ailoss\ailoss_crossentropy.c + "basic/base/ailoss/ailoss_crossentropy.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\base\ailoss\ailoss_crossentropy.h + "core/aifes_core.h" + "basic/base/ailayer/ailayer_softmax.h" + "basic/base/ailayer/ailayer_sigmoid.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\base\ailoss\ailoss_mse.c + "basic/base/ailoss/ailoss_mse.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\base\ailoss\ailoss_mse.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\base\aimath\aimath_basic.c + "basic/base/aimath/aimath_basic.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\base\aimath\aimath_f32.c + "basic/base/aimath/aimath_f32.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\base\aiopti\aiopti_adam.c + "basic/base/aiopti/aiopti_adam.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\base\aiopti\aiopti_adam.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\base\aiopti\aiopti_sgd.c + "basic/base/aiopti/aiopti_sgd.h" + "basic/base/aimath/aimath_basic.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\base\aiopti\aiopti_sgd.h + "core/aifes_core.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\cmsis\ailayer\ailayer_dense_cmsis.c + "basic/default/ailayer/ailayer_dense_default.h" + "../../../aifes.h" + "basic/cmsis/ailayer/ailayer_dense_cmsis.h" + "basic/cmsis/aimath/aimath_f32_cmsis.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\aifes.h + "core/aifes_math.h" + "core/aifes_core.h" + "basic/base/aimath/aimath_f32.h" + "basic/base/aimath/aimath_basic.h" + "basic/base/ailayer/ailayer_dense.h" + "basic/base/ailayer/ailayer_input.h" + "basic/base/ailayer/ailayer_relu.h" + "basic/base/ailayer/ailayer_leaky_relu.h" + "basic/base/ailayer/ailayer_elu.h" + "basic/base/ailayer/ailayer_sigmoid.h" + "basic/base/ailayer/ailayer_tanh.h" + "basic/base/ailayer/ailayer_softmax.h" + "basic/base/ailayer/ailayer_softsign.h" + "basic/base/ailoss/ailoss_mse.h" + "basic/base/ailoss/ailoss_crossentropy.h" + "basic/base/aiopti/aiopti_sgd.h" + "basic/base/aiopti/aiopti_adam.h" + "basic/default/aimath/aimath_f32_default.h" + "basic/default/ailayer/ailayer_dense_default.h" + "basic/default/ailayer/ailayer_input_default.h" + "basic/default/ailayer/ailayer_relu_default.h" + "basic/default/ailayer/ailayer_leaky_relu_default.h" + "basic/default/ailayer/ailayer_elu_default.h" + "basic/default/ailayer/ailayer_sigmoid_default.h" + "basic/default/ailayer/ailayer_tanh_default.h" + "basic/default/ailayer/ailayer_softmax_default.h" + "basic/default/ailayer/ailayer_softsign_default.h" + "basic/default/ailoss/ailoss_mse_default.h" + "basic/default/ailoss/ailoss_crossentropy_default.h" + "basic/default/aiopti/aiopti_sgd_default.h" + "basic/default/aiopti/aiopti_adam_default.h" + "basic/cmsis/aimath/aimath_f32_cmsis.h" + "basic/cmsis/ailayer/ailayer_dense_cmsis.h" + "basic/base/aialgo/aialgo_sequential_inference.h" + "basic/base/aialgo/aialgo_sequential_training.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\default\ailayer\ailayer_input_default.h + "basic/base/ailayer/ailayer_input.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\default\ailayer\ailayer_relu_default.h + "basic/base/ailayer/ailayer_relu.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\default\ailayer\ailayer_leaky_relu_default.h + "basic/base/ailayer/ailayer_leaky_relu.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\default\ailayer\ailayer_elu_default.h + "basic/base/ailayer/ailayer_elu.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\default\ailayer\ailayer_sigmoid_default.h + "basic/base/ailayer/ailayer_sigmoid.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\default\ailayer\ailayer_tanh_default.h + "basic/base/ailayer/ailayer_tanh.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\default\ailayer\ailayer_softmax_default.h + "basic/base/ailayer/ailayer_softmax.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\default\ailayer\ailayer_softsign_default.h + "basic/base/ailayer/ailayer_softsign.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\default\ailoss\ailoss_mse_default.h + "basic/base/ailoss/ailoss_mse.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\default\ailoss\ailoss_crossentropy_default.h + "basic/base/ailoss/ailoss_crossentropy.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\default\aiopti\aiopti_sgd_default.h + "basic/base/aiopti/aiopti_sgd.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\default\aiopti\aiopti_adam_default.h + "basic/base/aiopti/aiopti_adam.h" + "basic/default/aimath/aimath_f32_default.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\cmsis\aimath\aimath_f32_cmsis.h + "basic/base/aimath/aimath_f32.h" + "../../../aifes.h" + +1628682756 c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\cmsis\ailayer\ailayer_dense_cmsis.h + "../../../aifes.h" + "basic/base/ailayer/ailayer_dense.h" + "basic/cmsis/aimath/aimath_f32_cmsis.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\cmsis\aimath\aimath_f32_cmsis.c + "basic/cmsis/aimath/aimath_f32_cmsis.h" + "../../../aifes.h" + "arm_math.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\default\ailayer\ailayer_dense_default.c + "basic/default/ailayer/ailayer_dense_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\default\ailayer\ailayer_elu_default.c + "basic/default/ailayer/ailayer_elu_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\default\ailayer\ailayer_input_default.c + "basic/default/ailayer/ailayer_input_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\default\ailayer\ailayer_leaky_relu_default.c + "basic/default/ailayer/ailayer_leaky_relu_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\default\ailayer\ailayer_relu_default.c + "basic/default/ailayer/ailayer_relu_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\default\ailayer\ailayer_sigmoid_default.c + "basic/default/ailayer/ailayer_sigmoid_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\default\ailayer\ailayer_softmax_default.c + "basic/default/ailayer/ailayer_softmax_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\default\ailayer\ailayer_softsign_default.c + "basic/default/ailayer/ailayer_softsign_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\default\ailayer\ailayer_tanh_default.c + "basic/default/ailayer/ailayer_tanh_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\default\ailoss\ailoss_crossentropy_default.c + "basic/default/ailoss/ailoss_crossentropy_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\default\ailoss\ailoss_mse_default.c + "basic/default/ailoss/ailoss_mse_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\default\aimath\aimath_f32_default.c + "basic/default/aimath/aimath_f32_default.h" + + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\default\aiopti\aiopti_adam_default.c + "basic/default/aiopti/aiopti_adam_default.h" + +1628682756 source:c:\users\dk100\documents\aifes\github\aifes_for_arduino\examples\2_mnist_on_pc\3_mnist_retrain_keras\src\basic\default\aiopti\aiopti_sgd_default.c + "basic/default/aiopti/aiopti_sgd_default.h" + diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/MNIST_retrain_keras.layout b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/MNIST_retrain_keras.layout new file mode 100644 index 0000000..10b3d97 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/MNIST_retrain_keras.layout @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/keywords.txt b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/keywords.txt new file mode 100644 index 0000000..6d759a9 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/keywords.txt @@ -0,0 +1,217 @@ +####################################### +# Syntax Coloring Map For AIfES for Arduino +####################################### + +####################################### +# Datatypes (KEYWORD1) +####################################### + +aimodel_t KEYWORD1 +ailayer_t KEYWORD1 +ailoss_t KEYWORD1 +aiopti_t KEYWORD1 + +aicore_layertype_t KEYWORD1 +aicore_losstype_t KEYWORD1 +aicore_optitype_t KEYWORD1 + +ailayer_dense_t KEYWORD1 +ailayer_elu_t KEYWORD1 +ailayer_elu_f32_t KEYWORD1 +ailayer_input_t KEYWORD1 +ailayer_leaky_relu_t KEYWORD1 +ailayer_leaky_relu_f32_t KEYWORD1 +ailayer_relu_t KEYWORD1 +ailayer_sigmoid_t KEYWORD1 +ailayer_softmax_t KEYWORD1 +ailayer_softsign_t KEYWORD1 +ailayer_tanh_t KEYWORD1 + +ailoss_crossentropy_t KEYWORD1 +ailoss_mse_t KEYWORD1 + +aiopti_adam_t KEYWORD1 +aiopti_adam_f32_t KEYWORD1 +aiopti_adam_momentums_t KEYWORD1 +aiopti_sgd_t KEYWORD1 +aiopti_sgd_f32_t KEYWORD1 + +aitensor_t KEYWORD1 +####################################### +# Methods and Functions (KEYWORD2) +####################################### + +aialgo_backward_model KEYWORD2 +aialgo_calc_loss_model_f32 KEYWORD2 +aialgo_compile_model KEYWORD2 +aialgo_distribute_parameter_memory KEYWORD2 +aialgo_forward_model KEYWORD2 +aialgo_inference_model KEYWORD2 +aialgo_init_model_for_training KEYWORD2 +aialgo_print_loss_specs KEYWORD2 +aialgo_print_model_structure KEYWORD2 +aialgo_print_optimizer_specs KEYWORD2 +aialgo_schedule_inference_memory KEYWORD2 +aialgo_schedule_training_memory KEYWORD2 +aialgo_sizeof_inference_memory KEYWORD2 +aialgo_sizeof_parameter_memory KEYWORD2 +aialgo_sizeof_training_memory KEYWORD2 +aialgo_train_model KEYWORD2 +aialgo_update_params_model KEYWORD2 +aialgo_zero_gradients_model KEYWORD2 +ailayer_dense KEYWORD2 +ailayer_dense_backward KEYWORD2 +ailayer_dense_calc_result_shape KEYWORD2 +ailayer_dense_f32_cmsis KEYWORD2 +ailayer_dense_f32_default KEYWORD2 +ailayer_dense_forward KEYWORD2 +ailayer_dense_print_specs KEYWORD2 +ailayer_dense_set_paramem KEYWORD2 +ailayer_dense_set_trainmem KEYWORD2 +ailayer_dense_sizeof_paramem KEYWORD2 +ailayer_dense_sizeof_trainmem KEYWORD2 +ailayer_elu KEYWORD2 +ailayer_elu_backward KEYWORD2 +ailayer_elu_calc_result_shape KEYWORD2 +ailayer_elu_f32_default KEYWORD2 +ailayer_elu_forward KEYWORD2 +ailayer_elu_print_specs KEYWORD2 +ailayer_input KEYWORD2 +ailayer_input_backward KEYWORD2 +ailayer_input_calc_result_shape KEYWORD2 +ailayer_input_f32_default KEYWORD2 +ailayer_input_forward KEYWORD2 +ailayer_input_print_specs KEYWORD2 +ailayer_leaky_relu KEYWORD2 +ailayer_leaky_relu_backward KEYWORD2 +ailayer_leaky_relu_calc_result_shape KEYWORD2 +ailayer_leaky_relu_f32_default KEYWORD2 +ailayer_leaky_relu_forward KEYWORD2 +ailayer_leaky_relu_print_specs KEYWORD2 +ailayer_relu KEYWORD2 +ailayer_relu_backward KEYWORD2 +ailayer_relu_calc_result_shape KEYWORD2 +ailayer_relu_f32_default KEYWORD2 +ailayer_relu_forward KEYWORD2 +ailayer_relu_print_specs KEYWORD2 +ailayer_sigmoid KEYWORD2 +ailayer_sigmoid_backward KEYWORD2 +ailayer_sigmoid_calc_result_shape KEYWORD2 +ailayer_sigmoid_f32_default KEYWORD2 +ailayer_sigmoid_forward KEYWORD2 +ailayer_sigmoid_get_result_bound_f32_default KEYWORD2 +ailayer_sigmoid_print_specs KEYWORD2 +ailayer_softmax KEYWORD2 +ailayer_softmax_calc_result_shape KEYWORD2 +ailayer_softmax_f32_default KEYWORD2 +ailayer_softmax_forward KEYWORD2 +ailayer_softmax_print_specs KEYWORD2 +ailayer_softsign KEYWORD2 +ailayer_softsign_backward KEYWORD2 +ailayer_softsign_calc_result_shape KEYWORD2 +ailayer_softsign_f32_default KEYWORD2 +ailayer_softsign_forward KEYWORD2 +ailayer_softsign_print_specs KEYWORD2 +ailayer_tanh KEYWORD2 +ailayer_tanh_backward KEYWORD2 +ailayer_tanh_calc_result_shape KEYWORD2 +ailayer_tanh_f32_default KEYWORD2 +ailayer_tanh_forward KEYWORD2 +ailayer_tanh_get_result_bound_f32_default KEYWORD2 +ailayer_tanh_print_specs KEYWORD2 +ailayer_template KEYWORD2 +ailayer_template_backward KEYWORD2 +ailayer_template_calc_result_shape KEYWORD2 +ailayer_template_forward KEYWORD2 +ailayer_template_print_specs KEYWORD2 +ailayer_template_set_paramem KEYWORD2 +ailayer_template_set_trainmem KEYWORD2 +ailayer_template_sizeof_paramem KEYWORD2 +ailayer_template_sizeof_trainmem KEYWORD2 +ailoss_crossentropy KEYWORD2 +ailoss_crossentropy_calc_delta KEYWORD2 +ailoss_crossentropy_calc_loss KEYWORD2 +ailoss_crossentropy_dummy_backward KEYWORD2 +ailoss_crossentropy_f32_default KEYWORD2 +ailoss_crossentropy_print_specs KEYWORD2 +ailoss_mse KEYWORD2 +ailoss_mse_calc_delta KEYWORD2 +ailoss_mse_calc_loss KEYWORD2 +ailoss_mse_f32_default KEYWORD2 +ailoss_mse_print_specs KEYWORD2 +aimath_f32_cmsis_linear KEYWORD2 +aimath_f32_cmsis_mat_mul KEYWORD2 +aimath_f32_default_binary_crossentropy KEYWORD2 +aimath_f32_default_categorical_crossentropy KEYWORD2 +aimath_f32_default_categorical_crossentropy_sparse8 KEYWORD2 +aimath_f32_default_copy_tensor KEYWORD2 +aimath_f32_default_d_elu KEYWORD2 +aimath_f32_default_d_leaky_relu KEYWORD2 +aimath_f32_default_d_relu KEYWORD2 +aimath_f32_default_d_sigmoid KEYWORD2 +aimath_f32_default_d_softsign KEYWORD2 +aimath_f32_default_d_tanh KEYWORD2 +aimath_f32_default_divide KEYWORD2 +aimath_f32_default_elu KEYWORD2 +aimath_f32_default_expf_fast KEYWORD2 +aimath_f32_default_init_glorot_uniform KEYWORD2 +aimath_f32_default_init_he_uniform KEYWORD2 +aimath_f32_default_init_zeros KEYWORD2 +aimath_f32_default_leaky_relu KEYWORD2 +aimath_f32_default_linear KEYWORD2 +aimath_f32_default_mat_mul KEYWORD2 +aimath_f32_default_max KEYWORD2 +aimath_f32_default_min KEYWORD2 +aimath_f32_default_multiply KEYWORD2 +aimath_f32_default_norm_squared KEYWORD2 +aimath_f32_default_relu KEYWORD2 +aimath_f32_default_scalar_add KEYWORD2 +aimath_f32_default_scalar_mul KEYWORD2 +aimath_f32_default_sigmoid KEYWORD2 +aimath_f32_default_softmax KEYWORD2 +aimath_f32_default_softsign KEYWORD2 +aimath_f32_default_sqrt KEYWORD2 +aimath_f32_default_sum KEYWORD2 +aimath_f32_default_tanh KEYWORD2 +aimath_f32_default_tensor_add KEYWORD2 +aimath_f32_default_tensor_init_uniform KEYWORD2 +aimath_f32_default_tensor_sub KEYWORD2 +aimath_f32_default_tensor_sub_sparse8 KEYWORD2 +aimath_f32_default_transpose_vector KEYWORD2 +aimath_f32_default_zero_tensor KEYWORD2 +aimath_f32_print_aiscalar KEYWORD2 +aimath_f32_print_aitensor KEYWORD2 +aimath_sizeof_dtype KEYWORD2 +aimath_sizeof_tensor KEYWORD2 +aimath_sizeof_tensor_data KEYWORD2 +aimath_sizeof_tensor_params KEYWORD2 +aimath_tensor_elements KEYWORD2 +aimath_transpose_vector KEYWORD2 +aiopti_adam KEYWORD2 +aiopti_adam_f32_default KEYWORD2 +aiopti_adam_f32_default_begin_step KEYWORD2 +aiopti_adam_f32_default_end_step KEYWORD2 +aiopti_adam_init_optimem KEYWORD2 +aiopti_adam_print_specs KEYWORD2 +aiopti_adam_sizeof_optimem KEYWORD2 +aiopti_adam_update_params KEYWORD2 +aiopti_adam_zero_gradients KEYWORD2 +aiopti_sgd KEYWORD2 +aiopti_sgd_f32_default KEYWORD2 +aiopti_sgd_init_optimem_with_momentum KEYWORD2 +aiopti_sgd_init_optimem_without_momentum KEYWORD2 +aiopti_sgd_print_specs KEYWORD2 +aiopti_sgd_sizeof_optimem_with_momentum KEYWORD2 +aiopti_sgd_sizeof_optimem_without_momentum KEYWORD2 +aiopti_sgd_update_params_with_momentum KEYWORD2 +aiopti_sgd_update_params_without_momentum KEYWORD2 +aiopti_sgd_zero_gradients KEYWORD2 + +print_aiscalar KEYWORD2 +print_aitensor KEYWORD2 + +####################################### +# Constants (LITERAL1) +####################################### + +aif32 LITERAL1 diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/MNIST_to_cc.py b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/MNIST_to_cc.py new file mode 100644 index 0000000..fbfe515 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/MNIST_to_cc.py @@ -0,0 +1,81 @@ +import tensorflow as tf +from tensorflow.keras.utils import to_categorical + + +#Load and preprocess the MNIST data set +(x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data() + +x_train = x_train.astype(float)/255.0 +x_test = x_test.astype(float)/255.0 + +y_train = to_categorical(y_train) +y_test = to_categorical(y_test) + +print(x_train.shape) +print(x_test.shape) + +NUM_TRAINING_DATA = 30000 #Max 60000 +NUM_TEST_DATA = 10000 #Max 10000 + +def generate_train_data(): + ''' + Generates two header files. The first one contains as many training data according to the + number defined above. The second header file contains the labels belonging to the training + data. The training data and labels are both stored in a two-dimensional array. + ''' + + with open("MNIST_training_data.h", "w") as f: + f.write("float MNIST_training_data[" + str(NUM_TRAINING_DATA) + "][784] = {\n") + for i in range(0,NUM_TRAINING_DATA): + if i != 0: + f.write("},\n") + x_train_flatten = x_train[i].flatten() + f.write("{" + str(x_train_flatten[0]) + "f") + for j in range(1,784): + f.write(", " + str(x_train_flatten[j]) + "f") + f.write("}\n};") + + + with open("MNIST_training_data_label.h", "w") as f: + f.write("float MNIST_training_data_label[" + str(NUM_TRAINING_DATA) + "][10] = {\n") + for i in range(0,NUM_TRAINING_DATA): + if i != 0: + f.write("},\n") + f.write("{" + str(y_train[i][0]) + "f") + for j in range(1,10): + f.write(", " + str(y_train[i][j]) + "f") + f.write("}\n};") + + +def generate_test_data(): + ''' + Generates two header files. The first one contains as many test data according to the + number defined above. The second header file contains the labels belonging to the test + data. The test data and labels are both stored in a two-dimensional array. + ''' + + with open("MNIST_test_data.h", "w") as f: + f.write("float MNIST_test_data[" + str(NUM_TEST_DATA) + "][784] = {\n") + for i in range(0,NUM_TEST_DATA): + if i != 0: + f.write("},\n") + x_test_flatten = x_test[i].flatten() + f.write("{" + str(x_test_flatten[0]) + "f") + for j in range(1,784): + f.write(", " + str(x_test_flatten[j]) + "f") + f.write("}\n};") + + + with open("MNIST_test_data_label.h", "w") as f: + f.write("float MNIST_test_data_label[" + str(NUM_TEST_DATA) + "][10] = {\n") + for i in range(0,NUM_TEST_DATA): + if i != 0: + f.write("},\n") + f.write("{" + str(y_test[i][0]) + "f") + for j in range(1,10): + f.write(", " + str(y_test[i][j]) + "f") + f.write("}\n};") + + +generate_train_data() +generate_test_data() \ No newline at end of file diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/MNIST_weights.py b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/MNIST_weights.py new file mode 100644 index 0000000..141c6cb --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/MNIST_weights.py @@ -0,0 +1,98 @@ +import tensorflow as tf +from tensorflow.keras.utils import to_categorical + +from tensorflow.keras.models import Sequential +from tensorflow.keras.layers import Dense + +import cv2 + + +def train_model(): + ''' + Load and preprocess the MNIST data set and train a DNN. + ''' + + (x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data() + + x_train = x_train.astype(float)/255.0 + x_test = x_test.astype(float)/255.0 + + x_train = x_train.reshape(60000, 784) + x_test = x_test.reshape(10000, 784) + + y_train = to_categorical(y_train, 10) + y_test = to_categorical(y_test, 10) + + print(x_train.shape) + print(x_test.shape) + + + model = Sequential() + model.add(Dense(10, activation='relu', input_shape=(784,))) + model.add(Dense(10, activation='relu')) + model.add(Dense(10, activation='softmax')) + + model.summary() + + model.compile(optimizer="Adam", loss="categorical_crossentropy", metrics=["acc"]) + + model.fit(x_train, y_train, validation_split=0.2, epochs=1, batch_size=100) + + print("Accuracy on test data:") + model.evaluate(x_test, y_test) + + return model + + +def write_weights(model): + ''' + Load weights of the passed neural network. For each layer, the weights and the + bias are saved in a header file as an array. + ''' + + # Get the weights + weights = model.get_weights() + + # Write the weights for AIfES in a .h file + with open("MNIST_weights.h", "w") as f: + for k in range(0,len(weights)): + if (k%2) == 0: + f.write("float weights_data_dense_" + str(int(k/2)+1) + "[] = {") + else: + f.write("float bias_data_dense_" + str(int(k/2)+1) + "[] = {") + + if len(weights[k].shape) == 2: + for i in range(0,weights[k].shape[0]): + for j in range(0,weights[k].shape[1]): + if i == (weights[k].shape[0]-1) and j == (weights[k].shape[1]-1): + f.write(str(weights[k][i, j]) + "f") + else: + f.write(str(weights[k][i, j]) + "f, ") + f.write("};\n") + else: + for i in range(0,weights[k].shape[0]): + if i == (weights[k].shape[0]-1): + f.write(str(weights[k][i]) + "f") + else: + f.write(str(weights[k][i]) + "f, ") + f.write("};\n") + + f.write("\n") + + +def show_MNIST_test_img(i): + ''' + An image of the test data set can be displayed for verification. + ''' + + (_, _), (x_test, y_test) = tf.keras.datasets.mnist.load_data() + x_test = x_test.astype(float)/255.0 + img = cv2.resize(x_test[i], (200, 200)) + print(y_test[i]) + cv2.imshow('image',img) + cv2.waitKey(0) + + +mnist_model = train_model() +write_weights(mnist_model) +# show_MNIST_test_img(18) \ No newline at end of file diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/aifes.h b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/aifes.h new file mode 100644 index 0000000..e7af402 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/aifes.h @@ -0,0 +1,113 @@ +/** + * \file basic/aifes.h + * \internal + * \date 23.02.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Include all headers of the aifes 2 - basic module + * \details + */ + +#ifdef __cplusplus +extern "C" { +#endif + + +// Include AIfES core headers +#include "core/aifes_math.h" +#include "core/aifes_core.h" + +// Include the datatypes +#include "basic/base/aimath/aimath_f32.h" + +// Include basic datatype independent math functions +#include "basic/base/aimath/aimath_basic.h" + +// ---------------------------- Module base implementations ----------------------- +// ("abstract" super "classes". A hardware optimized implementation can "inherit" from these modules) + +// Include the layer base implementations +#include "basic/base/ailayer/ailayer_dense.h" +#include "basic/base/ailayer/ailayer_input.h" +#include "basic/base/ailayer/ailayer_relu.h" +#include "basic/base/ailayer/ailayer_leaky_relu.h" +#include "basic/base/ailayer/ailayer_elu.h" +#include "basic/base/ailayer/ailayer_sigmoid.h" +#include "basic/base/ailayer/ailayer_tanh.h" +#include "basic/base/ailayer/ailayer_softmax.h" +#include "basic/base/ailayer/ailayer_softsign.h" + +// Include the loss base implementations +#include "basic/base/ailoss/ailoss_mse.h" +#include "basic/base/ailoss/ailoss_crossentropy.h" + +// Include the optimizer base implementations +#include "basic/base/aiopti/aiopti_sgd.h" +#include "basic/base/aiopti/aiopti_adam.h" + +// ---------------------------- Module default implementations ----------------------- +// (Fallback functions if no hardware optimized implementation available) + +// Include the math in default implementation +#include "basic/default/aimath/aimath_f32_default.h" + +// Include the layers in default implementation +#include "basic/default/ailayer/ailayer_dense_default.h" +#include "basic/default/ailayer/ailayer_input_default.h" +#include "basic/default/ailayer/ailayer_relu_default.h" +#include "basic/default/ailayer/ailayer_leaky_relu_default.h" +#include "basic/default/ailayer/ailayer_elu_default.h" +#include "basic/default/ailayer/ailayer_sigmoid_default.h" +#include "basic/default/ailayer/ailayer_tanh_default.h" +#include "basic/default/ailayer/ailayer_softmax_default.h" +#include "basic/default/ailayer/ailayer_softsign_default.h" + +// Include the losses in default implementation +#include "basic/default/ailoss/ailoss_mse_default.h" +#include "basic/default/ailoss/ailoss_crossentropy_default.h" + +// Include the optimizers in default implementation +#include "basic/default/aiopti/aiopti_sgd_default.h" +#include "basic/default/aiopti/aiopti_adam_default.h" + +// ---------------------------- CMSIS implementations ----------------------- +// ATTENTION! +// If you want to use CMSIS for ARM controllers, you need to uncomment the define of AIFES_WITH_CMSIS + +//#define AIFES_WITH_CMSIS + +#ifdef AIFES_WITH_CMSIS + +// Include the math in cmsis implementation +#include "basic/cmsis/aimath/aimath_f32_cmsis.h" + +// Include the layers in cmsis implementation +#include "basic/cmsis/ailayer/ailayer_dense_cmsis.h" + +#endif /* AIFES_USE_CMSIS */ + +// ---------------------------- Algorithmic ----------------------- + +// Include the algorithmic +#include "basic/base/aialgo/aialgo_sequential_inference.h" +#include "basic/base/aialgo/aialgo_sequential_training.h" + +#ifdef __cplusplus +} // End extern "C" +#endif diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aialgo/aialgo_sequential_inference.c b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aialgo/aialgo_sequential_inference.c new file mode 100644 index 0000000..b11b867 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aialgo/aialgo_sequential_inference.c @@ -0,0 +1,224 @@ +/** + * \file basic/base/aialgo/aialgo_sequential_inference.c + * \version 2.0alpha + * \date 20.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/aialgo/aialgo_sequential_inference.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +#include +#include +#include + +uint32_t aialgo_sizeof_inference_memory(aimodel_t *model) +{ + uint16_t i; + uint32_t memory = 0, max_memory = 0; + ailayer_t *layer_ptr = model->input_layer; + + // Calculate result shapes and max amount of memory + for(i = 0; i < model->layer_count; i++) + { + layer_ptr->calc_result_shape(layer_ptr); + memory = aimath_sizeof_tensor_data(&(layer_ptr->result)); + if(memory > max_memory) max_memory = memory; + + layer_ptr = layer_ptr->output_layer; + } + + return 2 * max_memory; // Input and output buffer +} + +uint32_t aialgo_sizeof_parameter_memory(aimodel_t *model) +{ + int i; + ailayer_t *layer_ptr = model->input_layer; + uint32_t memory = 0; + + for(i = 0; i < model->layer_count; i++) + { + layer_ptr->calc_result_shape(layer_ptr); + + // Memory for the quantization parameter of the intermediate results + memory += layer_ptr->result.dtype->tensor_params_size; + + // Memory for trainable parameter + if(layer_ptr->sizeof_paramem != 0) + { + memory += layer_ptr->sizeof_paramem(layer_ptr); + } + + layer_ptr = layer_ptr->output_layer; + } + return memory; +} + +void aialgo_distribute_parameter_memory(aimodel_t *model, void *memory_ptr, uint32_t memory_size) +{ + int i; + ailayer_t *layer_ptr = model->input_layer; + uint32_t address_counter = 0; + + for(i = 0; i < model->layer_count; i++) + { + layer_ptr->calc_result_shape(layer_ptr); + + // Memory for the quantization parameter of the intermediate results + if(layer_ptr->result.dtype->tensor_params_size != 0){ + layer_ptr->result.tensor_params = memory_ptr + address_counter; + address_counter += layer_ptr->result.dtype->tensor_params_size; + } + + // Memory for trainable parameter + if(layer_ptr->sizeof_paramem != 0) + { + layer_ptr->set_paramem(layer_ptr, memory_ptr + address_counter); + address_counter += layer_ptr->sizeof_paramem(layer_ptr); + } + + layer_ptr = layer_ptr->output_layer; + } + return; +} + + +uint8_t aialgo_schedule_inference_memory(aimodel_t *model, void *memory_ptr, uint32_t memory_size) +{ + uint16_t i; + ailayer_t *layer_ptr = model->input_layer; + + // Init result tensor with shape and memory + for(i = 0; i < model->layer_count; i++) + { + layer_ptr->calc_result_shape(layer_ptr); + layer_ptr->result.data = memory_ptr + (i % 2) * memory_size / 2; + + layer_ptr = layer_ptr->output_layer; + } + + return 0; +} + +aitensor_t *aialgo_forward_model(aimodel_t *model, aitensor_t *input_data) +{ + uint16_t i; + ailayer_t *layer_ptr = model->input_layer; + + model->input_layer->result.data = input_data->data; + model->input_layer->result.tensor_params = input_data->tensor_params; + for(i = 0; i < model->layer_count; i++) + { + layer_ptr->forward(layer_ptr); + + // Print intermediate results + //print_aitensor(&layer_ptr->result); + + layer_ptr = layer_ptr->output_layer; + } + + return &(model->output_layer->result); +} + +aitensor_t *aialgo_inference_model(aimodel_t *model, aitensor_t *input_data, aitensor_t *output_data) +{ + uint32_t i, j; + + uint16_t input_batch_shape[input_data->dim]; + aitensor_t input_batch = { + .dtype = input_data->dtype, + .shape = input_batch_shape, + .dim = input_data->dim, + .tensor_params = input_data->tensor_params + }; + aitensor_t *output_batch; + + uint32_t input_multiplier = 1; + for(i = input_data->dim - 1; i > 0; i--) + { + input_multiplier *= input_data->shape[i]; + input_batch_shape[i] = input_data->shape[i]; + } + input_batch_shape[0] = 1; + + uint32_t output_multiplier = 1; + for(i = output_data->dim - 1; i > 0; i--) + { + output_multiplier *= output_data->shape[i]; + } + + for(i = 0; i < input_data->shape[0]; i++) + { + input_batch.data = input_data->data + i * input_multiplier * input_data->dtype->size; + + output_batch = aialgo_forward_model(model, &input_batch); + + // ToDo: Copy tensor + for(j = 0; j < aimath_tensor_elements(output_batch); j++) + { + memcpy(output_data->data + i * output_multiplier * input_data->dtype->size, + output_batch->data, + aimath_tensor_elements(output_batch) * input_data->dtype->size); + } + } + memcpy(output_data->tensor_params, output_batch->tensor_params, output_batch->dtype->tensor_params_size); + return output_data; +} + +uint8_t aialgo_compile_model(aimodel_t *model) +{ + ailayer_t *layer_ptr = model->input_layer; + uint16_t layer_counter = 1; + const uint16_t MAX_LAYER_COUNT = 128; // May be an other value + + model->trainable_params_count = 0; + while(layer_ptr != model->output_layer && layer_counter < MAX_LAYER_COUNT) + { + layer_counter++; + model->trainable_params_count += layer_ptr->trainable_params_count; + + layer_ptr = layer_ptr->output_layer; + } + model->layer_count = layer_counter; + + return 0; +} + +void aialgo_print_model_structure(aimodel_t *model) +{ + int i; + ailayer_t *layer_ptr = model->input_layer; + + printf("Layers:\n"); + for(i = 0; i < model->layer_count; i++){ + if(layer_ptr->layer_type->print_specs != 0){ + printf("%4d: %s (%s) <", i + 1, layer_ptr->layer_type->name, layer_ptr->result.dtype->name); + layer_ptr->layer_type->print_specs(layer_ptr, printf); + printf(">\n"); + } else { + printf("%4d: No specification found for this layer.\n", i + 1); + } + layer_ptr = layer_ptr->output_layer; + } + return; +} + diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aialgo/aialgo_sequential_inference.h b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aialgo/aialgo_sequential_inference.h new file mode 100644 index 0000000..7b448b9 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aialgo/aialgo_sequential_inference.h @@ -0,0 +1,145 @@ +/** + * \file basic/base/aialgo/aialgo_sequential_inference.h + * \internal + * \date 20.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Functions required for inference of models + * \details The functions target memory allocation/scheduling, the calculation of the forward pass and quantization for model inference + */ + +#ifndef AIALGO_SEQUENTIAL_INFERENCE +#define AIALGO_SEQUENTIAL_INFERENCE + +#include "core/aifes_core.h" +#include "core/aifes_math.h" +#include "basic/base/aimath/aimath_basic.h" +#include "basic/default/ailayer/ailayer_dense_default.h" + +/** @brief Calculate the memory requirements for intermediate results of an inference + * + * This memory is mainly for the result buffers of the layers. + * + * Use aialgo_schedule_inference_memory() to set the memory to the model. + * + * @param *model The model + * @return Required memory size in bytes + */ +uint32_t aialgo_sizeof_inference_memory(aimodel_t *model); + +/** @brief Calculate the memory requirements for the trainable parameters (like weights, bias, ...) of the model + * + * Use aialgo_distribute_parameter_memory() to set the memory to the model. + * + * @param *model The model + * @return Required memory size in bytes + */ +uint32_t aialgo_sizeof_parameter_memory(aimodel_t *model); + +/** @brief Assign the memory for intermediate results of an inference to the model + * + * The required memory size can be calculated with aialgo_sizeof_inference_memory() + * + * @param *model The model + * @param *memory_ptr Pointer to the memory block + * @param memory_size Size of the memory block (for error checking) + * @return 0 if successful + */ +uint8_t aialgo_schedule_inference_memory(aimodel_t *model, void *memory_ptr, uint32_t memory_size); + +/** @brief Assign the memory for the trainable parameters (like weights, bias, ...) of the model + * + * Only use this function if the parameters are not pre-trained or manually configured. + * Afterwards an initialization of the memory (for example by initializing the weights) has to be performed. + * + * The required memory size can be calculated with aialgo_sizeof_parameter_memory() + * + * @param *model The model + * @param *memory_ptr Pointer to the memory block + * @param memory_size Size of the memory block (for error checking) + * @return 0 if successful + */ +void aialgo_distribute_parameter_memory(aimodel_t *model, void *memory_ptr, uint32_t memory_size); + +/** @brief Perform a forward pass on the model + * + * The result is stored in the result tensor of the output layer and a pointer to this is returned. + * This output result is stored in the inference memory and is only valid as long as the inference memory is valid. + * To get the output as a separate tensor, use aialgo_inference_model() instead. + * + * @param *model The model + * @param *input_data Input data tensor of the same shape as the input_layer shape + * @return Pointer to the output data of the forward pass (points to the result tensor of the output layer) + */ +aitensor_t *aialgo_forward_model(aimodel_t *model, aitensor_t *input_data); + +/** @brief Perform an inference on the model / Run the model + * + * Make shure to initialize the model (aialgo_compile_model()) and schedule the inference memory + * (for example with aialgo_schedule_inference_memory() or aialgo_schedule_training_memory()) before + * calling this function. + * + * Example: + * \code{.c} + * float input_data[] = {0.0f, 1.0f}; + * uint16_t input_shape[] = {1, 2} + * aitensor_t input_tensor = { + * .dtype = aif32, + * .dim = 2, + * .shape = input_shape, + * .data = input_data + * }; + * + * float output_data[1]; + * uint16_t output_shape[] = {1, 1} + * aitensor_t output_tensor = { + * .dtype = aif32, + * .dim = 2, + * .shape = output_shape, + * .data = output_data + * }; + * + * aialgo_inference_model(&model, &input_tensor, &output_tensor); + * + * // The results are now in the output_tensor + * \endcode + * + * @param *model The model + * @param *input_data Input data tensor of the same shape as the input_layer shape + * @param *output_data Empty tensor for the results of the inference with the size of your outputs + * @return Pointer to the output_data tensor with the results + */ +aitensor_t *aialgo_inference_model(aimodel_t *model, aitensor_t *input_data, aitensor_t *output_data); + +/** @brief Initialize the model structure +* +* Counts the number of layers and trainable parameters in a model as preparation for inference or training. +* +* @param *model The model +* @return 0 if successful +*/ +uint8_t aialgo_compile_model(aimodel_t *model); + +/** @brief Print the layer structure of the model with the configured parameters +* +* @param *model The model +*/ +void aialgo_print_model_structure(aimodel_t *model); + +#endif // AIALGO_SEQUENTIAL_INFERENCE diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aialgo/aialgo_sequential_training.c b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aialgo/aialgo_sequential_training.c new file mode 100644 index 0000000..cae7d3f --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aialgo/aialgo_sequential_training.c @@ -0,0 +1,299 @@ +/** + * \file basic/base/aialgo/aialgo_sequential_training.c + * \version 2.0alpha + * \date 20.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/aialgo/aialgo_sequential_training.h" +#include "basic/base/aialgo/aialgo_sequential_inference.h" + +// ToDo: Remove dependency +#include "basic/default/aimath/aimath_f32_default.h" + +uint32_t aialgo_sizeof_training_memory(aimodel_t *model, aiopti_t *optimizer) +{ + uint16_t i, j; + ailayer_t *layer_ptr = model->input_layer; + uint32_t memory = 0; + + for(i = 0; i < model->layer_count; i++) + { + // Result memory + layer_ptr->calc_result_shape(layer_ptr); + memory += aimath_sizeof_tensor_data(&(layer_ptr->result)); + + // Memory for the qantization parameter of the deltas + if(layer_ptr->output_layer->deltas.dtype != 0){ + memory += layer_ptr->output_layer->deltas.dtype->tensor_params_size; + } + + // Trainingmemory e.g. for gradients + if(layer_ptr->sizeof_trainmem != 0) + { + memory += layer_ptr->sizeof_trainmem(layer_ptr); + } + + // optimization memory (e.g. first or second momentum) + if(optimizer->sizeof_optimem != 0){ + for(j = 0; j < layer_ptr->trainable_params_count; j++){ + memory += optimizer->sizeof_optimem(optimizer, layer_ptr->trainable_params[j]); + } + } + + layer_ptr = layer_ptr->output_layer; + } + return memory; +} + +uint8_t aialgo_schedule_training_memory(aimodel_t *model, aiopti_t *optimizer, void *memory_ptr, uint32_t memory_size) +{ + uint16_t i, j; + uint32_t address_counter = 0; + ailayer_t *layer_ptr = model->input_layer; + + for(i = 0; i < model->layer_count; i++) + { + // Result memory = deltas memory + layer_ptr->calc_result_shape(layer_ptr); + layer_ptr->result.data = memory_ptr + address_counter; + + layer_ptr->output_layer->deltas.dtype = layer_ptr->result.dtype; + layer_ptr->output_layer->deltas.dim = layer_ptr->result.dim; + layer_ptr->output_layer->deltas.shape = layer_ptr->result.shape; + layer_ptr->output_layer->deltas.data = memory_ptr + address_counter; + address_counter += aimath_sizeof_tensor_data(&(layer_ptr->result)); + + // Memory for the qantization parameter of the deltas + if(layer_ptr->output_layer->deltas.dtype != 0){ + layer_ptr->output_layer->deltas.tensor_params = memory_ptr + address_counter; + address_counter += layer_ptr->output_layer->deltas.dtype->tensor_params_size; + } + + // Training memory e.g. for gradients + if(layer_ptr->sizeof_trainmem != 0) + { + layer_ptr->set_trainmem(layer_ptr, memory_ptr + address_counter); + address_counter += layer_ptr->sizeof_trainmem(layer_ptr); + } + + // optimization memory (e.g. first or second momentum) + if(optimizer->sizeof_optimem != 0){ + for(j = 0; j < layer_ptr->trainable_params_count; j++){ + layer_ptr->optimem[j] = memory_ptr + address_counter; + address_counter += optimizer->sizeof_optimem(optimizer, layer_ptr->trainable_params[j]); + } + } + + layer_ptr = layer_ptr->output_layer; + } + + return 0; +} + +void aialgo_init_model_for_training(aimodel_t *model, aiopti_t *optimizer) +{ + uint16_t i, j; + ailayer_t *layer_ptr = model->input_layer; + + for(i = 0; i < model->layer_count; i++) + { + // Init the optimization memory (e.g. setting the momentums to zero) + for(j = 0; j < layer_ptr->trainable_params_count; j++){ + if(optimizer->init_optimem != 0){ + optimizer->init_optimem(optimizer, layer_ptr->trainable_params[j], layer_ptr->gradients[j], layer_ptr->optimem[j]); + } + } + layer_ptr = layer_ptr->output_layer; + } + return; +} + +void aialgo_backward_model(aimodel_t *model, aitensor_t *target_data) +{ + uint16_t i; + ailayer_t *layer_ptr = model->output_layer; + + model->loss->calc_delta(model->loss, target_data); + for(i = 0; i < model->layer_count; i++) + { +#ifdef DEBUG_CHECKS + if(layer_ptr->backward == 0){ + printf("\nError: No backward function implementation in layer %d\n", i); + return; + } +#endif + layer_ptr->backward(layer_ptr); + layer_ptr = layer_ptr->input_layer; + } + return; +} + +void aialgo_train_model(aimodel_t *model, aitensor_t *input_tensor, aitensor_t *target_tensor, aiopti_t *optimizer, uint32_t batch_size) +{ + uint32_t i; + + aitensor_t input_batch; + uint16_t input_batch_shape[input_tensor->dim]; + input_batch.dtype = input_tensor->dtype; + input_batch.dim = input_tensor->dim; + input_batch.shape = input_batch_shape; + input_batch.tensor_params = input_tensor->tensor_params; + aitensor_t target_batch; + uint16_t target_batch_shape[target_tensor->dim]; + target_batch.dtype = target_tensor->dtype; + target_batch.dim = target_tensor->dim; + target_batch.shape = target_batch_shape; + target_batch.tensor_params = target_tensor->tensor_params; + + uint32_t input_multiplier = 1; + for(i = input_tensor->dim - 1; i > 0; i--) + { + input_multiplier *= input_tensor->shape[i]; + input_batch_shape[i] = input_tensor->shape[i]; + } + input_multiplier *= input_tensor->dtype->size; + input_batch_shape[0] = 1; + uint32_t target_multiplier = 1; + for(i = target_tensor->dim - 1; i > 0; i--) + { + target_multiplier *= target_tensor->shape[i]; + target_batch_shape[i] = target_tensor->shape[i]; + } + target_multiplier *= target_tensor->dtype->size; + target_batch_shape[0] = 1; + + uint32_t batch_count = (uint32_t) (input_tensor->shape[0] / batch_size); + uint32_t batch; + for(batch = 0; batch < batch_count; batch++) + { + aialgo_zero_gradients_model(model, optimizer); + for(i = 0; i < batch_size; i++) + { + input_batch.data = input_tensor->data + batch * input_multiplier * batch_size + i * input_multiplier; + target_batch.data = target_tensor->data + batch * target_multiplier * batch_size + i * target_multiplier; + + aialgo_forward_model(model, &input_batch); + aialgo_backward_model(model, &target_batch); + } + aialgo_update_params_model(model, optimizer); + } + return; +} + +void aialgo_calc_loss_model_f32(aimodel_t *model, aitensor_t *input_tensor, aitensor_t *target_tensor, float *result) +{ + uint32_t i; + float loss; + + aitensor_t input_batch; + uint16_t input_batch_shape[input_tensor->dim]; + input_batch.dtype = input_tensor->dtype; + input_batch.dim = input_tensor->dim; + input_batch.shape = input_batch_shape; + input_batch.tensor_params = input_tensor->tensor_params; + aitensor_t target_batch; + uint16_t target_batch_shape[target_tensor->dim]; + target_batch.dtype = target_tensor->dtype; + target_batch.dim = target_tensor->dim; + target_batch.shape = target_batch_shape; + target_batch.tensor_params = target_tensor->tensor_params; + + uint32_t input_multiplier = 1; + for(i = input_tensor->dim - 1; i > 0; i--) + { + input_multiplier *= input_tensor->shape[i]; + input_batch_shape[i] = input_tensor->shape[i]; + } + input_multiplier *= input_tensor->dtype->size; + input_batch_shape[0] = 1; + uint32_t target_multiplier = 1; + for(i = target_tensor->dim - 1; i > 0; i--) + { + target_multiplier *= target_tensor->shape[i]; + target_batch_shape[i] = target_tensor->shape[i]; + } + target_multiplier *= target_tensor->dtype->size; + target_batch_shape[0] = 1; + + *result = 0; + for(i = 0; i < input_tensor->shape[0]; i++) + { + input_batch.data = input_tensor->data + i * input_multiplier; + target_batch.data = target_tensor->data + i * target_multiplier; + + aialgo_forward_model(model, &input_batch); + model->loss->calc_loss(model->loss, &target_batch, &loss); + *result += loss; + } + return; +} + +void aialgo_zero_gradients_model(aimodel_t *model, aiopti_t *optimizer) +{ + uint16_t i, j; + ailayer_t *layer_ptr = model->input_layer; + + for(i = 0; i < model->layer_count; i++) + { + for(j = 0; j < layer_ptr->trainable_params_count; j++){ + optimizer->zero_gradients(optimizer, layer_ptr->gradients[j]); + } + layer_ptr = layer_ptr->output_layer; + } + return; +} + +void aialgo_update_params_model(aimodel_t *model, aiopti_t *optimizer) +{ + uint16_t i, j; + ailayer_t *layer_ptr = model->input_layer; + + if(optimizer->begin_step != 0){ + optimizer->begin_step(optimizer); + } + for(i = 0; i < model->layer_count; i++) + { + for(j = 0; j < layer_ptr->trainable_params_count; j++){ + optimizer->update_params(optimizer, layer_ptr->trainable_params[j], layer_ptr->gradients[j], layer_ptr->optimem[j]); + } + layer_ptr = layer_ptr->output_layer; + } + if(optimizer->end_step != 0){ + optimizer->end_step(optimizer); + } + return; +} + +void aialgo_print_loss_specs(ailoss_t *loss) +{ + printf("%s (%s) <", loss->loss_type->name, loss->connection_layer.deltas.dtype->name); + loss->loss_type->print_specs(loss, printf); + printf(">"); + return; +} + +void aialgo_print_optimizer_specs(aiopti_t *opti) +{ + printf("%s (%s) <", opti->optimizer_type->name, opti->dtype->name); + opti->optimizer_type->print_specs(opti, printf); + printf(">"); + return; +} diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aialgo/aialgo_sequential_training.h b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aialgo/aialgo_sequential_training.h new file mode 100644 index 0000000..5ba2aa4 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aialgo/aialgo_sequential_training.h @@ -0,0 +1,148 @@ +/** + * \file basic/base/aialgo/aialgo_sequential_training.h + * \internal + * \date 20.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Functions required for the training of models + * \details The functions target memory allocation/scheduling and the backpropagation for model training + */ + +#ifndef AIALGO_SEQUENTIAL_TRAINING +#define AIALGO_SEQUENTIAL_TRAINING + +#include "core/aifes_core.h" +#include "core/aifes_math.h" +#include "basic/base/aimath/aimath_basic.h" + +/** @brief Calculate the memory requirements for model training + * + * This memory is used for intermediate results, gradients and momentums. + * + * Use aialgo_schedule_training_memory() to set the memory to the model. + * + * @param *model The model + * @param *optimizer The optimizer that is used for training + * @return Required memory size in bytes + */ +uint32_t aialgo_sizeof_training_memory(aimodel_t *model, aiopti_t *optimizer); + +/** @brief Assign the memory for model training + * + * This memory is used for intermediate results, gradients and momentums. + * + * The required memory size can be calculated with aialgo_sizeof_training_memory(). + * + * @param *model The model + * @param *optimizer The optimizer that is used for training + * @param *memory_ptr Pointer to the memory block + * @param memory_size Size of the memory block (for error checking) + * @return 0 if successful + */ +uint8_t aialgo_schedule_training_memory(aimodel_t *model, aiopti_t *optimizer, void *memory_ptr, uint32_t memory_size); + +/** @brief Initialize the optimization memory of the model layers + * + * @param *model The model + * @param *optimizer The optimizer that is used for training + */ +void aialgo_init_model_for_training(aimodel_t *model, aiopti_t *optimizer); + +/** @brief Perform the backward pass + * + * @param *model The model + * @param *target_data The tensor containing the target data / labels + */ +void aialgo_backward_model(aimodel_t *model, aitensor_t *target_data); + +/** @brief Perform one training epoch on all data batches of the dataset using backpropagation + * + * Make shure to initialize the model (aialgo_compile_model()) and schedule the training memory + * (for example with aialgo_schedule_training_memory()) and initialize the training memory + * (aialgo_init_model_for_training()) before calling this function. + * + * Example: Training of an F32 model for multiple epochs + * \code{.c} + * int epochs = 100; + * int batch_size = 4; + * int print_interval = 10; + * + * float loss; + * for(i = 0; i < epochs; i++) + * { + * // One epoch of training. Iterates through the whole data once + * aialgo_train_model(&model, &input_tensor, &target_tensor, optimizer, batch_size); + + * // Calculate and print loss every print_interval epochs + * if(i % print_interval == 0) + * { + * aialgo_calc_loss_model_f32(&model, &input_tensor, &target_tensor, &loss); + * printf("Epoch %5d: loss: %f\n", i, loss); + * } + * } + * \endcode + * + * @param *model The model + * @param *input_tensor The tensor containing the input data + * @param *target_tensor The tensor containing the target data / labels + * @param *optimizer The optimizer that is used for training + * @param batch_size Size of a batch / Number of input vektors + */ +void aialgo_train_model(aimodel_t *model, aitensor_t *input_tensor, aitensor_t *target_tensor, aiopti_t *optimizer, uint32_t batch_size); + +/** @brief Calculate the loss in \link aimath_f32.h F32 \endlink data type + * + * @param *model The model + * @param *input_data Tensor containing the input data + * @param *target_data Tensor containing the target data / labels + * @param *result The calculated loss will be written here + */ +void aialgo_calc_loss_model_f32(aimodel_t *model, aitensor_t *input_data, aitensor_t *target_data, float *result); + +/** @brief Set the gradients to zero + * + * @param *model The model + * @param *optimizer The optimizer that is used for training + */ +void aialgo_zero_gradients_model(aimodel_t *model, aiopti_t *optimizer); + +/** @brief Perform the optimization step on the model parameters + * + * @param *model The model + * @param *optimizer The optimizer that is used for training + */ +void aialgo_update_params_model(aimodel_t *model, aiopti_t *optimizer); + +/** @brief Print the loss specs + * + * Prints information like type, data type and constants to the console. + * + * @param *loss The loss + */ +void aialgo_print_loss_specs(ailoss_t *loss); + +/** @brief Print the optimizer specs + * + * Prints information like type, data type and constants to the console. + * + * @param *opti The optimizer + */ +void aialgo_print_optimizer_specs(aiopti_t *opti); + +#endif // AIALGO_SEQUENTIAL_TRAINING diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_dense.c b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_dense.c new file mode 100644 index 0000000..d3306b4 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_dense.c @@ -0,0 +1,246 @@ +/** + * \file basic/base/ailayer/ailayer_dense.c + * \version 2.0alpha + * \date 20.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/ailayer/ailayer_dense.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_layertype_t ailayer_dense_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "Dense", + .print_specs = ailayer_dense_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_layertype_t *ailayer_dense_type = &ailayer_dense_type_s; + +ailayer_t *ailayer_dense(ailayer_dense_t *layer, ailayer_t *input_layer) +{ + layer->base.layer_type = ailayer_dense_type; + + layer->base.input_layer = input_layer; + input_layer->output_layer = &(layer->base); + + layer->base.layer_configuration = layer; + layer->base.result.dtype = layer->result_dtype; + layer->base.result.dim = 2; + layer->base.result.shape = layer->result_shape; + layer->base.result.shape[1] = layer->neurons; + + layer->base.deltas.dtype = layer->result_dtype; + layer->base.deltas.dim = 2; + layer->base.deltas.shape = input_layer->result.shape; + + layer->weights.dim = 2; + layer->weights.dtype = layer->weights_dtype; + layer->weights.shape = layer->weights_shape; + layer->weights.shape[0] = input_layer->result.shape[1]; + layer->weights.shape[1] = layer->neurons; + + layer->bias.dim = 2; + layer->bias.dtype = layer->bias_dtype; + layer->bias.shape = layer->bias_shape; + layer->bias.shape[0] = 1; + layer->bias.shape[1] = layer->neurons; + + layer->base.forward = ailayer_dense_forward; + layer->base.backward = ailayer_dense_backward; + + layer->base.calc_result_shape = ailayer_dense_calc_result_shape; + layer->base.sizeof_paramem = ailayer_dense_sizeof_paramem; + layer->base.set_paramem = ailayer_dense_set_paramem; + layer->base.sizeof_trainmem = ailayer_dense_sizeof_trainmem; + layer->base.set_trainmem = ailayer_dense_set_trainmem; + + layer->base.get_result_bound = 0; + + layer->base.trainable_params_count = 2; + layer->base.trainable_params = layer->trainable_params; + layer->base.gradients = layer->gradients; + layer->base.optimem = layer->optimem; + + layer->trainable_params[0] = &layer->weights; + layer->trainable_params[1] = &layer->bias; + + return &layer->base; +} + +void ailayer_dense_forward(ailayer_t *self) +{ + aitensor_t *input_tensor = &(self->input_layer->result); + aitensor_t *result_tensor = &(self->result); + ailayer_dense_t *layer = (ailayer_dense_t *)(self->layer_configuration); + aitensor_t *weight_tensor = &(layer->weights); + aitensor_t *bias_tensor = &(layer->bias); + + // z = x * W + b + layer->linear(input_tensor, weight_tensor, bias_tensor, result_tensor); + + return; +} + + +void ailayer_dense_backward(ailayer_t *self) +{ + aitensor_t *delta_in = &(self->deltas); + aitensor_t *delta_out = &(self->output_layer->deltas); + aitensor_t *x_in = &(self->input_layer->result); + ailayer_dense_t *layer = (ailayer_dense_t *)(self->layer_configuration); + aitensor_t *weights = &(layer->weights); + aitensor_t *d_weights = layer->gradients[0]; + aitensor_t *d_bias = layer->gradients[1]; + + int8_t temp_result_data[aimath_sizeof_tensor_data(d_weights)]; + aitensor_t temp_result = { + .dim = 2, + .shape = d_weights->shape, + .data = temp_result_data, + .dtype = d_weights->dtype, + .tensor_params = d_weights->tensor_params + }; + + aimath_transpose_vector(x_in); + // d_weights += x_in^T * delta_out + layer->mat_mul(x_in, delta_out, &temp_result); + layer->tensor_add(d_weights, &temp_result, d_weights); + aimath_transpose_vector(x_in); + // d_bias += delta_out + layer->tensor_add(d_bias, delta_out, d_bias); + + // Calculate delta for next layer. Do not before calculating gradients!!! May override x_in!!! + // d_in = w^T * d_out = (d_out^T * w)^T + aimath_transpose_vector(delta_out); + aimath_transpose_vector(delta_in); + layer->mat_mul(weights, delta_out, delta_in); + aimath_transpose_vector(delta_in); + aimath_transpose_vector(delta_out); + + return; +} + +void ailayer_dense_calc_result_shape(ailayer_t *self) +{ + ailayer_dense_t *layer = (ailayer_dense_t *)(self->layer_configuration); + + self->result.shape[0] = self->input_layer->result.shape[0]; + self->result.shape[1] = layer->neurons; + + return; +} + +uint32_t ailayer_dense_sizeof_paramem(const ailayer_t *self) +{ + uint32_t memory = 0; + ailayer_dense_t *layer = (ailayer_dense_t *)(self->layer_configuration); + + // Weights + memory += layer->weights_dtype->tensor_params_size; + memory += self->input_layer->result.shape[1] * layer->neurons * aimath_sizeof_dtype(layer->weights_dtype); // data + + // Bias + memory += layer->bias_dtype->tensor_params_size; + memory += layer->neurons * aimath_sizeof_dtype(layer->bias_dtype); // data + return memory; +} + +void ailayer_dense_set_paramem(ailayer_t *self, void *memory_ptr) +{ + uint32_t address_counter = 0; + ailayer_dense_t *layer = (ailayer_dense_t *) (self->layer_configuration); + + layer->weights.tensor_params = memory_ptr + address_counter; + address_counter += layer->weights_dtype->tensor_params_size; + layer->weights.dim = 2; + layer->weights.dtype = layer->weights_dtype; + layer->weights.shape = layer->weights_shape; + layer->weights.shape[0] = self->input_layer->result.shape[1]; + layer->weights.shape[1] = layer->neurons; + layer->weights.data = memory_ptr + address_counter; + address_counter += aimath_sizeof_tensor_data(&(layer->weights)); + + layer->bias.tensor_params = memory_ptr + address_counter; + address_counter += layer->bias_dtype->tensor_params_size; + layer->bias.dim = 2; + layer->bias.dtype = layer->bias_dtype; + layer->bias.shape = layer->bias_shape; + layer->bias.shape[0] = 1; + layer->bias.shape[1] = layer->neurons; + layer->bias.data = memory_ptr + address_counter; + //address_counter += aimath_sizeof_tensor_data(&(configuration->bias)); + + layer->trainable_params[0] = &(layer->weights); + layer->trainable_params[1] = &(layer->bias); + + return; +} + +uint32_t ailayer_dense_sizeof_trainmem(const ailayer_t *self) +{ + uint32_t memory = 0; + ailayer_dense_t *layer = (ailayer_dense_t *)(self->layer_configuration); + + memory += aimath_sizeof_tensor(&layer->weights); + memory += aimath_sizeof_tensor(&layer->bias); + return memory; +} + +void ailayer_dense_set_trainmem(ailayer_t *self, void *memory_ptr) +{ + uint32_t address_counter = 0; + ailayer_dense_t *layer = (ailayer_dense_t *) (self->layer_configuration); + + // Weights gradients in gradients[0] + self->gradients[0] = memory_ptr; + address_counter += sizeof(aitensor_t); + self->gradients[0]->data = memory_ptr + address_counter; + self->gradients[0]->dtype = layer->weights.dtype; + self->gradients[0]->dim = 2; + self->gradients[0]->shape = layer->weights.shape; + address_counter += aimath_sizeof_tensor_data(layer->gradients[0]); + self->gradients[0]->tensor_params = memory_ptr + address_counter; + address_counter += aimath_sizeof_tensor_params(layer->gradients[0]); + + // Bias gradients in gradients[1] + self->gradients[1] = memory_ptr + address_counter; + address_counter += sizeof(aitensor_t); + self->gradients[1]->data = memory_ptr + address_counter; + self->gradients[1]->dtype = layer->bias.dtype; + self->gradients[1]->dim = 2; + self->gradients[1]->shape = layer->bias.shape; + address_counter += aimath_sizeof_tensor_data(layer->gradients[1]); + self->gradients[1]->tensor_params = memory_ptr + address_counter; + address_counter += aimath_sizeof_tensor_params(layer->gradients[1]); + + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void ailayer_dense_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)) +{ + ailayer_dense_t *layer = (ailayer_dense_t *)(self->layer_configuration); + + print("neurons: %ld", (long unsigned int) layer->neurons); +} +#endif diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_dense.h b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_dense.h new file mode 100644 index 0000000..66cbd2d --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_dense.h @@ -0,0 +1,296 @@ +/** + * \file basic/base/ailayer/ailayer_dense.h + * \internal + * \date 20.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link ailayer layer \endlink implementation of the Dense layer + * + * This is an "abstract" data-type independent implementation. To use the layer use one of the provided + * implementations for a specific hardware and data-type (for example from ailayer_dense_default.h) or set + * the required math functions on your own. + * + * \image html ailayer_dense_schematic.png width=200px + * + * The Dense layer (or fully connected layer) is the core layer of a FNN and calculates the weighted sums of its inputs + * @f[ + * y = x \cdot W + b + * @f] + * with the number of inputs \f$ K \f$, the number of neurons / outputs \f$ M \f$, + * the input vektor \f$ x \in \mathbb{R}^{1 \times K} \f$, the output vektor \f$ y \in \mathbb{R}^{1 \times M} \f$, + * the weights matrix \f$ W \in \mathbb{R}^{K \times M} \f$ and the bias \f$ b \in \mathbb{R}^{1 \times M} \f$. + * + * To increase the computational speed it is also possible to process a whole data batch at ones: + * @f[ + * Y = X \cdot W \oplus b + * @f] + * with the number of inputs \f$ K \f$, the number of neurons / outputs \f$ M \f$, the batch size \f$ N \f$, + * the input matrix \f$ X \in \mathbb{R}^{N \times K} \f$, the output vektor \f$ Y \in \mathbb{R}^{N \times M} \f$, + * the weights matrix \f$ W \in \mathbb{R}^{K \times M} \f$ and the bias \f$ b \in \mathbb{R}^{1 \times M} \f$. + * In this case the \f$ \oplus \f$ means a broadcasted addition of the bias to the rows of the result of \f$ X \cdot W \f$: + * @f[ + * X \cdot W \oplus b \rightarrow X \cdot W + \left( \begin{array}{c} 1 \\ \vdots \\ 1 \\ \end{array}\right) \cdot b + * @f] + * Keep in mind that also the memory size for intermediate results increases with the batch size. + * + * The results of the forward pass of this layer are written to the result tensor of the base ailayer_t struct. + */ + +#ifndef AILAYER_DENSE +#define AILAYER_DENSE + +#include "core/aifes_core.h" +#include "basic/default/aimath/aimath_f32_default.h" + +#define DENSE_WEIGHTS_SIZE(INPUTS, OUTPUTS) ((INPUTS) * (OUTPUTS)) +#define DENSE_BIAS_SIZE(OUTPUTS) (OUTPUTS) + +#define DENSE_WEIGHTS_SHAPE(INPUTS, OUTPUTS) {INPUTS, OUTPUTS} +#define DENSE_BIAS_SHAPE(OUTPUTS) {1, OUTPUTS} + +typedef struct ailayer_dense ailayer_dense_t; + +/** @brief General \link ailayer_dense.h Dense layer \endlink structure +* +*/ +struct ailayer_dense { + ailayer_t base; /**< Inherited field members from general ailayer struct. */ + const aimath_dtype_t *result_dtype; /**< Data type of the inference result values. */ + + /** @name Layer configuration + * @brief Required configuration parameters for the layer + * + * These fields have to be configured by the user before calling the initializer function. + */ + ///@{ + uint32_t neurons; /**< Layer neurons count (number of outputs). */ + ///@} + + /** @name Trainable parameters + * @brief Data fields for the trainable parameters (weights, bias) of the layer + */ + ///@{ + aitensor_t weights; /**< Tensor containing the layer weights. */ + aitensor_t bias; /**< Tensor containing the layer bias weights. */ + + const aimath_dtype_t *weights_dtype; /**< Data type of the weights. */ + const aimath_dtype_t *bias_dtype; /**< Data type of the bias weights. */ + + uint16_t weights_shape[2]; /**< Weights tensor shape (n x m matrix). */ + uint16_t bias_shape[2]; /**< Bias weights tensor shape (n x m matrix). */ + + aitensor_t *trainable_params[2]; /**< Pointer to the weights and biases (which are the trainable parameters). */ + aitensor_t *gradients[2]; /**< Gradients structure for the back propagation algorithm. */ + void *optimem[2]; /**< Memory field used by the trainings optimizer. */ + ///@} + + /** @name Math functions + * @brief Required data type specific math functions + */ + ///@{ + + /** @brief Required math function: Linear transformation + * + * Requires a math function that performs a linear transformation:\n + * @f[ + * result = a \cdot b \oplus c = a \cdot b + \left( \begin{array}{c} 1 \\ \vdots \\ 1 \\ \end{array}\right) \cdot c + * @f] + * + * @param a Matrix with dimension \f$ N \times K \f$ (input) + * @param b Matrix with dimension \f$ K \times M \f$ (input) + * @param c Laying vektor with dimension \f$ 1 \times M \f$ (input) + * @param result Matrix with dimension \f$ N \times M \f$ (output) + */ + void (*linear)(const aitensor_t *a, const aitensor_t *b, const aitensor_t *c, aitensor_t *result); + + /** @brief Required math function: Matrix multiplication + * + * Requires a math function that performs a matrix multiplication on two 2D tensors:\n + * @f[ + * result = a \cdot b + * @f] + * + * @param a Matrix with dimension \f$ N \times K \f$ (input) + * @param b Matrix with dimension \f$ K \times M \f$ (input) + * @param result Matrix with dimension \f$ N \times M \f$ (output) + */ + void (*mat_mul)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + /** @brief Required math function: Element wise tensor addition + * + * Requires a math function that adds two tensors element wise:\n + * @f[ + * result = a + b + * @f] + */ + void (*tensor_add)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + ///@} + + uint16_t result_shape[2]; /**< Inference result tensor (ailayer.result) shape. */ +}; + +/** @brief Dense layer type + * + * Defines the type of the layer (for example for type checks and debug prints). + * See aicore_layertype for more information about the layer type. + */ +extern const aicore_layertype_t *ailayer_dense_type; + +/** @brief Initialize and connect the given Dense layer + * + * This function represents the "constructor" of the abstract Dense layer. It initializes the layer structure + * and connects it to the previous layer.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example ailayer_dense_f32_default()). + * + * @param *layer The layer to initialize. + * @param *input_layer The previous layer that provides the inputs to the layer. + * @return Pointer to the (successfully) initialized general layer structure (ailayer_dense.base) + */ +ailayer_t *ailayer_dense(ailayer_dense_t *layer, ailayer_t *input_layer); + +/** @brief Calculate the forward pass for given Dense layer + * + * *Implementation of ailayer.forward.* + * + * It uses the result tensor of the previous layer as input and writes the result of the forward pass + * to the result tensor (ailayer.result) of the given layer. + * + * Calculation of the forward pass result: + * @f[ + * x_{out} \leftarrow x_{in} \cdot w \oplus b + * @f] + * + * \f$ w \f$: Weights matrix\n + * \f$ b \f$: Bias vektor\n + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ x_{out} \f$: Result of the forward pass of this layer\n\n + * + * Used math functions: + * * ailayer_dense.linear + * + * @param *self Layer to calculate the forward path for. + */ +void ailayer_dense_forward(ailayer_t *self); + +/** @brief Calculate the backward pass for the given Dense layer + * + * *Implementation of ailayer.backward.* + * + * It uses the deltas tensor of the next layer as input and writes the result of the backward pass + * to the deltas tensor (ailayer.deltas) of the given layer. + * + * Calculation of the gradients: + * @f[ + * \partial w \leftarrow \partial w + x_{in}^T \cdot \delta_{out} + * @f] + * @f[ + * \partial b \leftarrow \partial b + \left( \begin{array}{c} 1 & \cdots & 1 \\ \end{array}\right) \cdot \delta_{out} + * @f] + * + * Calculation of the errors for the previous layer: + * @f[ + * \delta_{in} \leftarrow w^T \cdot \delta_{out} = (\delta_{out}^T \cdot w)^T + * @f] + * + * \f$ w \f$: Weights matrix\n + * \f$ b \f$: Bias vektor\n + * \f$ \partial w \f$: Gradients matrix for the weights\n + * \f$ \partial b \f$: Gradients vektor for the bias\n + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ \delta_{in} \f$: Result of the backward pass of this layer\n + * \f$ \delta_{out} \f$: Result of the backward pass of the next layer\n\n + * + * Used math functions: + * * ailayer_dense.mat_mul + * * ailayer_dense.tensor_add + * + * @param *self Layer to calculate the backward path for. + */ +void ailayer_dense_backward(ailayer_t *self); + +/** @brief Calculate the shape of the result tensor (ailayer.result) + * + * *Implementation of ailayer.calc_result_shape.* + * + * Resulting shape is [count_inputs x count_neurons] + * + * @param *self Layer to calculate the resulting shape for. + */ +void ailayer_dense_calc_result_shape(ailayer_t *self); + + +/** @brief Calculate and return the parameter memory size needed for this layer + * + * *Implementation of ailayer.sizeof_paramem.* + * + * The parameter size is calculated for the \link ailayer_dense.weights weights \endlink and + * \link ailayer_dense.bias bias \endlink tensors, including the data and tensor_params fields. + * + * @param *self The layer to calculate the parameter memory size for + * @return Calculated parameter memory size in bytes. + */ +uint32_t ailayer_dense_sizeof_paramem(const ailayer_t *self); + +/** @brief Distribute provided memory to the parameter pointers + * + * *Implementation of ailayer.set_paramem.* + * + * Distributes the given buffer to the parameter pointers and sets + * the tensor parameters for weights and bias holding structures.\n + * The required parameter size can be calculated with ailayer_dense_sizeof_paramem() + * + * @param *self The layer to set the memory fields for. + * @param *memory_ptr The memory that can be used for the parameters + */ +void ailayer_dense_set_paramem(ailayer_t *self, void *memory_ptr); + +/** @brief Calculate and return the memory size needed by this layer for training + * + * *Implementation of ailayer.sizeof_trainmem.* + * + * The memory size is calculated for the gradient tensors of weights and bias. + * + * @param *self The layer to calculate the gradient memory size for. + * @return Calculated gradient memory size in bytes. + */ +uint32_t ailayer_dense_sizeof_trainmem(const ailayer_t *self); + +/** @brief Distribute provided memory to the gradients pointers + * + * *Implementation of ailayer.set_trainmem.* + * + * The required memory size can be calculated with ailayer_dense_sizeof_trainmem(). + * + * @param *self The layer to set the memory fields for. + * @param *memory_ptr The memory that can be used for the gradients + */ +void ailayer_dense_set_trainmem(ailayer_t *self, void *memory_ptr); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the layer specification + * + * @param *self The layer to print the specification for + * @param *print Pointer to the print function to use + */ +void ailayer_dense_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // AILAYER_DENSE diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_elu.c b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_elu.c new file mode 100644 index 0000000..35aaa1f --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_elu.c @@ -0,0 +1,113 @@ +/** + * \file basic/base/ailayer/ailayer_elu.c + * \version 2.0alpha + * \date 16.03.2021 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/ailayer/ailayer_elu.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_layertype_t ailayer_elu_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "ELU", + .print_specs = ailayer_elu_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_layertype_t *ailayer_elu_type = &ailayer_elu_type_s; + +ailayer_t *ailayer_elu(ailayer_elu_t *layer, ailayer_t *input_layer) ////const void *beta1, +{ + layer->base.layer_type = ailayer_elu_type; + + layer->base.input_layer = input_layer; + input_layer->output_layer = &(layer->base); + + layer->base.layer_configuration = layer; + layer->base.result.dtype = layer->dtype; + layer->base.result.shape = input_layer->result.shape; + layer->base.result.dim = input_layer->result.dim; + + layer->base.deltas.dtype = layer->dtype; + layer->base.deltas.dim = 2; + layer->base.deltas.shape = layer->base.result.shape; + + layer->base.forward = ailayer_elu_forward; + layer->base.backward = ailayer_elu_backward; + + layer->base.calc_result_shape = ailayer_elu_calc_result_shape; + layer->base.sizeof_paramem = 0; + layer->base.set_paramem = 0; + layer->base.sizeof_trainmem = 0; + layer->base.set_trainmem = 0; + + layer->base.trainable_params_count = 0; + + return &(layer->base); +} + +void ailayer_elu_forward(ailayer_t *self) +{ + ailayer_elu_t *layer = (ailayer_elu_t *)(self->layer_configuration); + aitensor_t *x_in = &(self->input_layer->result); + aitensor_t *x_out = &(self->result); + + layer->elu(x_in, layer->alpha, x_out); + return; +} + + +void ailayer_elu_backward(ailayer_t *self) +{ + ailayer_elu_t *layer = (ailayer_elu_t *)(self->layer_configuration); + aitensor_t *delta_in = &(self->deltas); + aitensor_t *delta_out = &(self->output_layer->deltas); + aitensor_t *x_in = &(self->input_layer->result); + + // delta_in = delta_out .* elu'(x_in) + layer->d_elu(x_in, layer->alpha, delta_in); + layer->multiply(delta_in, delta_out, delta_in); + return; +} + +void ailayer_elu_calc_result_shape(ailayer_t *self) +{ + /* Unused: Shape is already defined (Pointer) + + self->result.shape[0] = self->input_layer->result.shape[0]; + self->result.shape[1] = self->input_layer->result.shape[1]; + */ + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void ailayer_elu_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)) +{ + ailayer_elu_t *self_casted = (ailayer_elu_t *) self->layer_configuration; + + print("alpha: "); + self_casted->dtype->print_aiscalar(self_casted->alpha, print); + return; +} +#endif + diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_elu.h b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_elu.h new file mode 100644 index 0000000..4a5bde3 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_elu.h @@ -0,0 +1,200 @@ +/** + * \file basic/base/ailayer/ailayer_elu.h + * \internal + * \date 15.03.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link ailayer layer \endlink implementation of the ELU activation layer + * + * This is an "abstract" data-type independent implementation. To use the layer use one of the provided + * implementations for a specific hardware and data-type (for example from ailayer_elu_default.h) or set + * the required math functions on your own. + * + * \image html ailayer_elu_schematic.png width=200px + * + * The ELU layer is used as an activation function layer right after a dense layer. It calculates + * @f[ + * y = \begin{cases} + \alpha (e^x - 1) & \text{if } x \leq 0\\ + x & \text{if } x > 0 + \end{cases} + * @f] + * for every element of the input tensor. + * + * The results of the forward pass of this layer are written to the result tensor of the base ailayer_t struct. + */ + +#ifndef ELU_LAYER +#define ELU_LAYER + +#include "core/aifes_core.h" + +typedef struct ailayer_elu ailayer_elu_t; + +/** @brief General \link ailayer_elu.h ELU layer \endlink struct +* +*/ +struct ailayer_elu { + ailayer_t base; /**< Inherited field members from general ailayer struct. */ + const aimath_dtype_t *dtype; /**< Data type of the input and inference result values. */ + + /** @name Layer configuration + * @brief Required configuration parameters for the layer + * + * These fields have to be configured by the user before calling the initializer function. + */ + ///@{ + void *alpha; /**< Parameter \f$ \alpha \f$ used to calculate ELU function for input values < 0. */ + ///@} + + /** @name Math functions + * @brief Required data type specific math functions + */ + ///@{ + + /** @brief Required math function: ELU + * + * Requires a math function that calculates the element wise ELU of a tensor:\n + * @f[ + * result_{i} = \begin{cases} + \alpha \cdot (e^{x_i} - 1) & \text{if } x_i < 0 \\ + x_i & \text{if } x_i \geq 0 + \end{cases} + * @f] + * + * @param x N-dimensional tensor (input) + * @param result N-dimensional tensor (output) + */ + void (*elu)(const aitensor_t *x, const void *alpha, aitensor_t *result); + + /** @brief Required math function: Derivative of ELU + * + * Requires a math function that calculates the element wise ELU derivative of a tensor:\n + * @f[ + * result_{i} = \begin{cases} + \alpha \cdot e^{x_i} & \text{if } x_i < 0\\ + 1 & \text{if } x_i \geq 0 + \end{cases} + * @f] + * + * @param x N-dimensional tensor (input) + * @param result N-dimensional tensor (output) + */ + void (*d_elu)(const aitensor_t *x, const void *alpha, aitensor_t *result); + + /** @brief Required math function: Element wise tensor multiplication + * + * Requires a math function that multiplies two tensors element wise:\n + * @f[ + * result = a \circ b + * @f] + */ + void (*multiply)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + ///@} +}; + +/** @brief ELU layer type + * + * Defines the type of the layer (for example for type checks and debug prints). + * See aicore_layertype for more information about the layer type. + */ +extern const aicore_layertype_t *ailayer_elu_type; + +/** @brief Initialize and connect the given ELU layer + * + * This function represents the "constructor" of the abstract ELU layer. It initializes the layer structure + * and connects it to the previous layer.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example ailayer_elu_f32_default()). + * + * @param *layer The layer to initialize. + * @param *input_layer The previous layer that provides the inputs to the layer. + * @return Pointer to the (successfully) initialized general layer structure (ailayer_elu.base) + */ +ailayer_t *ailayer_elu(ailayer_elu_t *layer, ailayer_t *input_layer); + +/** @brief Calculate the forward pass for given ELU layer + * + * *Implementation of ailayer.forward.* + * + * It uses the result tensor of the previous layer as input and writes the result of the forward pass + * to the result tensor (ailayer.result) of the given layer. + * + * Calculation of the forward pass result: + * @f[ + * x_{out} \leftarrow ELU(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ x_{out} \f$: Result of the forward pass of this layer\n\n + * + * Used math functions: + * * ailayer_elu.elu + * + * @param *self Layer to calculate the forward path for. + */ +void ailayer_elu_forward(ailayer_t *self); + +/** @brief Calculate the backward pass for the given ELU layer + * + * *Implementation of ailayer.backward.* + * + * It uses the deltas tensor of the next layer as input and writes the result of the backward pass + * to the deltas tensor (ailayer.deltas) of the given layer. + * + * Calculation of the errors for the previous layer: + * @f[ + * \delta_{in} \leftarrow \delta_{out} \circ ELU'(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ \delta_{in} \f$: Result of the backward pass of this layer\n + * \f$ \delta_{out} \f$: Result of the backward pass of the next layer\n\n + * + * Used math functions: + * * ailayer_elu.elu + * * ailayer_elu.d_elu + * * ailayer_elu.multiply + * + * @param *self Layer to calculate the backward path for. + */ +void ailayer_elu_backward(ailayer_t *self); + +/** @brief Calculate the shape of the result tensor + * + * *Implementation of ailayer.calc_result_shape.* + * + * As the result tensor shape is shared with the result tensor shape of the previous layer (no change in shape is needed), + * this function returns without doing anything. + * + * @param *self Layer to calculate the resulting shape for. + */ +void ailayer_elu_calc_result_shape(ailayer_t *self); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the layer specification + * + * @param *self The layer to print the specification for + * @param *print Pointer to the print function to use + */ +void ailayer_elu_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // ELU_LAYER diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_input.c b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_input.c new file mode 100644 index 0000000..7c8213e --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_input.c @@ -0,0 +1,99 @@ +/** + * \file basic/base/ailayer/ailayer_input.c + * \version 2.0alpha + * \date 20.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/ailayer/ailayer_input.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_layertype_t ailayer_input_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "Input", + .print_specs = ailayer_input_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_layertype_t *ailayer_input_type = &ailayer_input_type_s; + + +ailayer_t *ailayer_input(ailayer_input_t *layer) +{ + layer->base.layer_type = ailayer_input_type; + + layer->base.layer_configuration = layer; + layer->base.result.dtype = layer->dtype; + layer->base.result.shape = layer->input_shape; + layer->base.result.dim = layer->input_dim; + + layer->base.deltas.dtype = 0; + + layer->base.forward = ailayer_input_forward; + layer->base.backward = ailayer_input_backward; + + layer->base.calc_result_shape = ailayer_input_calc_result_shape; + layer->base.sizeof_paramem = 0; + layer->base.set_paramem = 0; + layer->base.sizeof_trainmem = 0; + layer->base.set_trainmem = 0; + + layer->base.get_result_bound = 0; + + layer->base.trainable_params_count = 0; + + return &layer->base; +} + +void ailayer_input_forward(ailayer_t *self) +{ + return; +} + +void ailayer_input_backward(ailayer_t *self) +{ + return; +} + +void ailayer_input_calc_result_shape(ailayer_t *self) +{ + //ailayer_input_t *layer = (ailayer_input_t *)(self->layer_configuration); + + //self->result.shape[0] = configuration->input_shape[0]; + //self->result.shape[1] = configuration->input_shape[1]; + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void ailayer_input_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)) +{ + ailayer_input_t *self_casted = (ailayer_input_t *) self; + uint8_t i = 0; + + print("Dim: %d; Shape: [%d", self_casted->input_dim, self_casted->input_shape[i]); + for(i = 1; i < self_casted->input_dim; i++){ + print(", %d", self_casted->input_shape[i]); + } + print("]"); + return; +} +#endif diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_input.h b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_input.h new file mode 100644 index 0000000..66ed268 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_input.h @@ -0,0 +1,120 @@ +/** + * \file ailayer/ailayer_input.h + * \internal + * \date 20.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link ailayer layer \endlink implementation of the Input layer + * + * This is an "abstract" data-type independent implementation. To use the layer use one of the provided + * implementations for a specific hardware and data-type (for example from ailayer_input_default.h) or set + * the required math functions on your own. + * + * The Input layer is the start layer for every AIfES 2 model. + */ + +#ifndef INPUT_LAYER +#define INPUT_LAYER + +#include "core/aifes_core.h" + +typedef struct ailayer_input ailayer_input_t; + +/** @brief General \link ailayer_input.h Input layer \endlink structure +* +*/ +struct ailayer_input { + ailayer_t base; /**< Inherited field members from general ailayer struct. */ + const aimath_dtype_t *dtype; /**< Data type of the input and inference result values. */ + + /** @name Layer configuration + * @brief Required configuration parameters for the layer + * + * These fields have to be configured by the user before calling the initializer function. + */ + ///@{ + uint8_t input_dim; /**< Dimension of the input tensor. */ + uint16_t *input_shape; /**< Shape of the input tensor. */ + ///@} +}; + +/** @brief Input layer type + * + * Defines the type of the layer (for example for type checks and debug prints). + * See aicore_layertype for more information about the layer type. + */ +extern const aicore_layertype_t *ailayer_input_type; + +/** @brief Initialize the given Input layer + * + * This function represents the "constructor" of the abstract Input layer. It initializes the layer structure + * and connects it to the previous layer.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example ailayer_input_f32_default()). + * + * @param *layer The layer to initialize. + * @return Pointer to the (successfully) initialized general layer structure (ailayer_input.base). + */ +ailayer_t *ailayer_input(ailayer_input_t *layer); + +/** @brief Calculate the forward pass for given Input layer + * + * *Implementation of ailayer.forward.* + * + * The standard layer interface demands a forward pass function + * for the net scheduler, but theres no sense in 'calculating' + * an input layer, so this function returns without doing something. + * + * @param *self Layer to 'calculate' the forward pass for. + */ +void ailayer_input_forward(ailayer_t *self); + +/** @brief Calculate the backward pass for the given Input layer + * + * *Implementation of ailayer.backward.* + * + * The standard layer interface demands a backward pass function + * for the net scheduler, but theres no sense in 'calculating' + * an input layer, so this function returns without doing something. + * + * @param *self Layer to 'calculate' the backward pass for. + */ +void ailayer_input_backward(ailayer_t *self); + +/** @brief Calculate the shape of the result tensor + * + * *Implementation of ailayer.calc_result_shape* + * + * As the result shape is given by the configured ailayer_input.input_shape parameter, + * this function returns without doing anything. + * + * @param *self Layer to calculate the resulting shape for. + */ +void ailayer_input_calc_result_shape(ailayer_t *self); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the layer specification + * + * @param *self The layer to print the specification for + * @param *print Pointer to the print function to use + */ +void ailayer_input_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // INPUT_LAYER diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_leaky_relu.c b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_leaky_relu.c new file mode 100644 index 0000000..0f73b81 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_leaky_relu.c @@ -0,0 +1,112 @@ +/** + * \file basic/base/ailayer/ailayer_leaky_relu.c + * \version 2.0alpha + * \date 16.03.2021 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/ailayer/ailayer_leaky_relu.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_layertype_t ailayer_leaky_relu_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "Leaky ReLU", + .print_specs = ailayer_leaky_relu_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_layertype_t *ailayer_leaky_relu_type = &ailayer_leaky_relu_type_s; + +ailayer_t *ailayer_leaky_relu(ailayer_leaky_relu_t *layer, ailayer_t *input_layer) ////const void *beta1, +{ + layer->base.layer_type = ailayer_leaky_relu_type; + + layer->base.input_layer = input_layer; + input_layer->output_layer = &(layer->base); + layer->base.layer_configuration = layer; + layer->base.result.dtype = layer->dtype; + layer->base.result.shape = input_layer->result.shape; + layer->base.result.dim = input_layer->result.dim; + + layer->base.deltas.dtype = layer->dtype; + layer->base.deltas.dim = 2; + layer->base.deltas.shape = layer->base.result.shape; + + layer->base.forward = ailayer_leaky_relu_forward; + layer->base.backward = ailayer_leaky_relu_backward; + + layer->base.calc_result_shape = ailayer_leaky_relu_calc_result_shape; + layer->base.sizeof_paramem = 0; + layer->base.set_paramem = 0; + layer->base.sizeof_trainmem = 0; + layer->base.set_trainmem = 0; + + layer->base.trainable_params_count = 0; + + return &(layer->base); +} + +void ailayer_leaky_relu_forward(ailayer_t *self) +{ + ailayer_leaky_relu_t *layer = (ailayer_leaky_relu_t *)(self->layer_configuration); + aitensor_t *x_in = &(self->input_layer->result); + aitensor_t *x_out = &(self->result); + + layer->leaky_relu(x_in, layer->alpha, x_out); + return; +} + +void ailayer_leaky_relu_backward(ailayer_t *self) +{ + ailayer_leaky_relu_t *layer = (ailayer_leaky_relu_t *)(self->layer_configuration); + aitensor_t *delta_in = &(self->deltas); + aitensor_t *delta_out = &(self->output_layer->deltas); + aitensor_t *x_in = &(self->input_layer->result); + + // delta_in = delta_out .* relu'(x_in) + layer->d_leaky_relu(x_in, layer->alpha, delta_in); + layer->multiply(delta_in, delta_out, delta_in); + + return; +} + +void ailayer_leaky_relu_calc_result_shape(ailayer_t *self) +{ + /* Unused: Shape is already defined (Pointer) + + self->result.shape[0] = self->input_layer->result.shape[0]; + self->result.shape[1] = self->input_layer->result.shape[1]; + */ + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void ailayer_leaky_relu_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)) +{ + ailayer_leaky_relu_t *self_casted = (ailayer_leaky_relu_t *) self; + + print("alpha: "); + self_casted->dtype->print_aiscalar(self_casted->alpha, print); + return; +} +#endif + diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_leaky_relu.h b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_leaky_relu.h new file mode 100644 index 0000000..c012ea2 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_leaky_relu.h @@ -0,0 +1,200 @@ +/** + * \file basic/base/ailayer/ailayer_leaky_relu.h + * \internal + * \date 15.03.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link ailayer layer \endlink implementation of the Leaky ReLU activation layer + * + * This is an "abstract" data-type independent implementation. To use the layer use one of the provided + * implementations for a specific hardware and data-type (for example from ailayer_leaky_relu_default.h) or set + * the required math functions on your own. + * + * \image html ailayer_leaky_relu_schematic.png width=200px + * + * The Leaky ReLU layer is used as an activation function layer right after a dense layer. It calculates + * @f[ + * y = \begin{cases} + \alpha x & \text{if } x < 0\\ + x & \text{if } x \geq 0 + \end{cases} + * @f] + * for every element of the input tensor. + * + * The results of the forward pass of this layer are written to the result tensor of the base ailayer_t struct. + */ + +#ifndef LEAKY_RELU_LAYER +#define LEAKY_RELU_LAYER + +#include "core/aifes_core.h" + +typedef struct ailayer_leaky_relu ailayer_leaky_relu_t; + +/** @brief General \link ailayer_leaky_relu.h Leaky ReLU layer \endlink struct +* +*/ +struct ailayer_leaky_relu { + ailayer_t base; /**< Inherited field members from general ailayer struct. */ + const aimath_dtype_t *dtype; /**< Data type of the input and inference result values. */ + + /** @name Layer configuration + * @brief Required configuration parameters for the layer + * + * These fields have to be configured by the user before calling the initializer function. + */ + ///@{ + void *alpha; /**< Parameter \f$ \alpha \f$ used to calculate Leaky ReLU function for input values < 0. */ + ///@} + + /** @name Math functions + * @brief Required data type specific math functions + */ + ///@{ + + /** @brief Required math function: Leaky ReLU + * + * Requires a math function that calculates the element wise Leaky ReLU of a tensor:\n + * @f[ + * result_{i} = \begin{cases} + \alpha \cdot x_i & \text{if } x_i < 0 \\ + x_i & \text{if } x_i \geq 0 + \end{cases} + * @f] + * + * @param x N-dimensional tensor (input) + * @param result N-dimensional tensor (output) + */ + void (*leaky_relu)(const aitensor_t *x, const void *alpha, aitensor_t *result); + + /** @brief Required math function: Derivative of Leaky ReLU + * + * Requires a math function that calculates the element wise Leaky ReLU derivative of a tensor:\n + * @f[ + * result_{i} = \begin{cases} + \alpha & \text{if } x_i < 0\\ + 1 & \text{if } x_i \geq 0 + \end{cases} + * @f] + * + * @param x N-dimensional tensor (input) + * @param result N-dimensional tensor (output) + */ + void (*d_leaky_relu)(const aitensor_t *x, const void *alpha, aitensor_t *result); + + /** @brief Required math function: Element wise tensor multiplication + * + * Requires a math function that multiplies two tensors element wise:\n + * @f[ + * result = a \circ b + * @f] + */ + void (*multiply)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + ///@} +}; + +/** @brief Leaky ReLU layer type + * + * Defines the type of the layer (for example for type checks and debug prints). + * See aicore_layertype for more information about the layer type. + */ +extern const aicore_layertype_t *ailayer_leaky_relu_type; + +/** @brief Initialize and connect the given Leaky ReLU layer + * + * This function represents the "constructor" of the abstract Leaky ReLU layer. It initializes the layer structure + * and connects it to the previous layer.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example ailayer_leaky_relu_f32_default()). + * + * @param *layer The layer to initialize. + * @param *input_layer The previous layer that provides the inputs to the layer. + * @return Pointer to the (successfully) initialized general layer structure (ailayer_leaky_relu.base). + */ +ailayer_t *ailayer_leaky_relu(ailayer_leaky_relu_t *layer, ailayer_t *input_layer); + +/** @brief Calculate the forward pass for given Leaky ReLU layer + * + * *Implementation of ailayer.forward.* + * + * It uses the result tensor of the previous layer as input and writes the result of the forward pass + * to the result tensor (ailayer.result) of the given layer. + * + * Calculation of the forward pass result: + * @f[ + * x_{out} \leftarrow LeakyReLU(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ x_{out} \f$: Result of the forward pass of this layer\n\n + * + * Used math functions: + * * ailayer_leaky_relu.leaky_relu + * + * @param *self Layer to calculate the forward path for. + */ +void ailayer_leaky_relu_forward(ailayer_t *self); + +/** @brief Calculate the backward pass for the given Leaky ReLU layer + * + * *Implementation of ailayer.backward.* + * + * It uses the deltas tensor of the next layer as input and writes the result of the backward pass + * to the deltas tensor (ailayer.deltas) of the given layer. + * + * Calculation of the errors for the previous layer: + * @f[ + * \delta_{in} \leftarrow \delta_{out} \circ LeakyReLU'(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ \delta_{in} \f$: Result of the backward pass of this layer\n + * \f$ \delta_{out} \f$: Result of the backward pass of the next layer\n\n + * + * Used math functions: + * * ailayer_leaky_relu.leaky_relu + * * ailayer_leaky_relu.d_leaky_relu + * * ailayer_leaky_relu.multiply + * + * @param *self Layer to calculate the backward path for. + */ +void ailayer_leaky_relu_backward(ailayer_t *self); + +/** @brief Calculate the shape of the result tensor + * + * *Implementation of ailayer.calc_result_shape.* + * + * As the result tensor shape is shared with the result tensor shape of the previous layer (no change in shape is needed), + * this function returns without doing anything. + * + * @param *self Layer to calculate the resulting shape for. + */ +void ailayer_leaky_relu_calc_result_shape(ailayer_t *self); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the layer specification + * + * @param *self The layer to print the specification for + * @param *print Pointer to the print function to use + */ +void ailayer_leaky_relu_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // LEAKY_RELU_LAYER diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_relu.c b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_relu.c new file mode 100644 index 0000000..e252381 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_relu.c @@ -0,0 +1,110 @@ +/** + * \file basic/base/ailayer/ailayer_relu.c + * \version 2.0alpha + * \date 07.12.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/ailayer/ailayer_relu.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_layertype_t ailayer_relu_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "ReLU", + .print_specs = ailayer_relu_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_layertype_t *ailayer_relu_type = &ailayer_relu_type_s; + + +ailayer_t *ailayer_relu(ailayer_relu_t *layer, ailayer_t *input_layer) +{ + layer->base.layer_type = ailayer_relu_type; + + layer->base.input_layer = input_layer; + input_layer->output_layer = &(layer->base); + + layer->base.layer_configuration = layer; + layer->base.result.dtype = layer->dtype; + layer->base.result.shape = input_layer->result.shape; + layer->base.result.dim = input_layer->result.dim; + + layer->base.deltas.dtype = layer->dtype; + layer->base.deltas.dim = 2; + layer->base.deltas.shape = layer->base.result.shape; + + layer->base.forward = ailayer_relu_forward; + layer->base.backward = ailayer_relu_backward; + + layer->base.calc_result_shape = ailayer_relu_calc_result_shape; + layer->base.sizeof_paramem = 0; + layer->base.set_paramem = 0; + layer->base.sizeof_trainmem = 0; + layer->base.set_trainmem = 0; + + layer->base.trainable_params_count = 0; + + return &(layer->base); +} + +void ailayer_relu_forward(ailayer_t *self) +{ + ailayer_relu_t *layer = (ailayer_relu_t *)(self->layer_configuration); + aitensor_t *x_in = &(self->input_layer->result); + aitensor_t *x_out = &(self->result); + + layer->relu(x_in, x_out); + return; +} + + +void ailayer_relu_backward(ailayer_t *self) +{ + ailayer_relu_t *layer = (ailayer_relu_t *)(self->layer_configuration); + aitensor_t *delta_in = &(self->deltas); + aitensor_t *delta_out = &(self->output_layer->deltas); + aitensor_t *x_in = &(self->input_layer->result); + + // delta_in = delta_out .* relu'(x_in) + layer->d_relu(x_in, delta_in); + layer->multiply(delta_in, delta_out, delta_in); + return; +} + +void ailayer_relu_calc_result_shape(ailayer_t *self) +{ + /* Unused: Shape is already defined (Pointer) + + self->result.shape[0] = self->input_layer->result.shape[0]; + self->result.shape[1] = self->input_layer->result.shape[1]; + */ + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void ailayer_relu_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)) +{ + return; +} +#endif + diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_relu.h b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_relu.h new file mode 100644 index 0000000..469aeaf --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_relu.h @@ -0,0 +1,187 @@ + +/** + * \file basic/base/ailayer/ailayer_relu.h + * \internal + * \date 07.12.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link ailayer layer \endlink implementation of the ReLU activation layer + * + * This is an "abstract" data-type independent implementation. To use the layer use one of the provided + * implementations for a specific hardware and data-type (for example from ailayer_relu_default.h) or set + * the required math functions on your own. + * + * \image html ailayer_relu_schematic.png width=200px + * + * The ReLU layer is used as an activation function layer right after a dense layer. It calculates + * @f[ + * y = \begin{cases} + 0 & \text{if } x < 0\\ + x & \text{if } x \geq 0 + \end{cases} + * @f] + * for every element of the input tensor. + * + * The results of the forward pass of this layer are written to the result tensor of the base ailayer_t struct. + */ + +#ifndef RELU_LAYER +#define RELU_LAYER + +#include "core/aifes_core.h" + +typedef struct ailayer_relu ailayer_relu_t; + +/** @brief General \link ailayer_relu.h ReLU layer \endlink struct +* +*/ +struct ailayer_relu { + ailayer_t base; /**< Inherited field members from general ailayer struct. */ + const aimath_dtype_t *dtype; /**< Data type of the input and inference result values. */ + + /** @name Math functions + * @brief Required data type specific math functions + */ + ///@{ + + /** @brief Required math function: ReLU + * + * Requires a math function that calculates the element wise ReLU of a tensor:\n + * @f[ + * result_{i} = max(0, x_{i}) + * @f] + * + * @param x N-dimensional tensor (input) + * @param result N-dimensional tensor (output) + */ + void (*relu)(const aitensor_t *x, aitensor_t *result); + + /** @brief Required math function: Derivative of ReLU + * + * Requires a math function that calculates the element wise ReLU derivative of a tensor:\n + * @f[ + * result_{i} = \begin{cases} + 0 & \text{if } x_i < 0\\ + 1 & \text{if } x_i \geq 0 + \end{cases} + * @f] + * + * @param x N-dimensional tensor (input) + * @param result N-dimensional tensor (output) + */ + void (*d_relu)(const aitensor_t *x, aitensor_t *result); + + /** @brief Required math function: Element wise tensor multiplication + * + * Requires a math function that multiplies two tensors element wise:\n + * @f[ + * result = a \circ b + * @f] + */ + void (*multiply)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); +}; + +/** @brief ReLU layer type + * + * Defines the type of the layer (for example for type checks and debug prints). + * See aicore_layertype for more information about the layer type. + */ +extern const aicore_layertype_t *ailayer_relu_type; + +/** @brief Initialize and connect the given ReLU layer + * + * This function represents the "constructor" of the abstract ReLU layer. It initializes the layer structure + * and connects it to the previous layer.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example ailayer_relu_f32_default()). + * + * @param *layer The layer to initialize. + * @param *input_layer The previous layer that provides the inputs to the layer. + * @return Pointer to the (successfully) initialized general layer structure (ailayer_relu.base). + */ +ailayer_t *ailayer_relu(ailayer_relu_t *layer, ailayer_t *input_layer); + +/** @brief Calculate the forward pass for given ReLU layer + * + * *Implementation of ailayer.forward.* + * + * It uses the result tensor of the previous layer as input and writes the result of the forward pass + * to the result tensor (ailayer.result) of the given layer. + * + * Calculation of the forward pass result: + * @f[ + * x_{out} \leftarrow ReLU(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ x_{out} \f$: Result of the forward pass of this layer\n\n + * + * Used math functions: + * * ailayer_relu.relu + * + * @param *self Layer to calculate the forward path for. + */ +void ailayer_relu_forward(ailayer_t *self); + +/** @brief Calculate the backward pass for the given ReLU layer + * + * *Implementation of ailayer.backward.* + * + * It uses the deltas tensor of the next layer as input and writes the result of the backward pass + * to the deltas tensor (ailayer.deltas) of the given layer. + * + * Calculation of the errors for the previous layer: + * @f[ + * \delta_{in} \leftarrow \delta_{out} \circ ReLU'(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ \delta_{in} \f$: Result of the backward pass of this layer\n + * \f$ \delta_{out} \f$: Result of the backward pass of the next layer\n\n + * + * Used math functions: + * * ailayer_relu.relu + * * ailayer_relu.d_relu + * * ailayer_relu.multiply + * + * @param *self Layer to calculate the backward path for. + */ +void ailayer_relu_backward(ailayer_t *self); + +/** @brief Calculate the shape of the result tensor + * + * *Implementation of ailayer.calc_result_shape.* + * + * As the result tensor shape is shared with the result tensor shape of the previous layer (no change in shape is needed), + * this function returns without doing anything. + * + * @param *self Layer to calculate the resulting shape for. + */ +void ailayer_relu_calc_result_shape(ailayer_t *self); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the layer specification + * + * @param *self The layer to print the specification for + * @param *print Pointer to the print function to use + */ +void ailayer_relu_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // RELU_LAYER diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_sigmoid.c b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_sigmoid.c new file mode 100644 index 0000000..21b9252 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_sigmoid.c @@ -0,0 +1,122 @@ +/** + * \file basic/base/ailayer/ailayer_sigmoid.c + * \version 2.0alpha + * \date 20.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/ailayer/ailayer_sigmoid.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_layertype_t ailayer_sigmoid_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "Sigmoid", + .print_specs = ailayer_sigmoid_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_layertype_t *ailayer_sigmoid_type = &ailayer_sigmoid_type_s; + + +ailayer_t *ailayer_sigmoid(ailayer_sigmoid_t *layer, ailayer_t *input_layer) +{ + layer->base.layer_type = ailayer_sigmoid_type; + + layer->base.input_layer = input_layer; + input_layer->output_layer = &(layer->base); + + layer->base.layer_configuration = layer; + layer->base.result.dtype = layer->dtype; + layer->base.result.dim = input_layer->result.dim; + layer->base.result.shape = input_layer->result.shape; + + layer->base.deltas.dtype = layer->dtype; + layer->base.deltas.dim = 2; + layer->base.deltas.shape = layer->base.result.shape; + + layer->base.forward = ailayer_sigmoid_forward; + layer->base.backward = ailayer_sigmoid_backward; + + layer->base.calc_result_shape = ailayer_sigmoid_calc_result_shape; + layer->base.sizeof_paramem = 0; + layer->base.set_paramem = 0; + layer->base.sizeof_trainmem = 0; + layer->base.set_trainmem = 0; + + layer->base.trainable_params_count = 0; + + return &(layer->base); +} + +void ailayer_sigmoid_forward(ailayer_t *self) +{ + ailayer_sigmoid_t *layer = (ailayer_sigmoid_t *)(self->layer_configuration); + aitensor_t *x_in = &(self->input_layer->result); + aitensor_t *x_out = &(self->result); + + layer->sigmoid(x_in, x_out); + return; +} + + +void ailayer_sigmoid_backward(ailayer_t *self) +{ + ailayer_sigmoid_t *layer = (ailayer_sigmoid_t *)(self->layer_configuration); + aitensor_t *delta_in = &(self->deltas); + aitensor_t *delta_out = &(self->output_layer->deltas); + aitensor_t *x_in = &(self->input_layer->result); + + int8_t temp_result_params[aimath_sizeof_tensor_params(x_in) + 1]; // +1 to prevent array of size 0 + int8_t temp_result_data[aimath_sizeof_tensor_data(x_in)]; + aitensor_t temp_result = { + .dim = 2, + .shape = x_in->shape, + .data = temp_result_data, + .dtype = x_in->dtype, + .tensor_params = temp_result_params + }; + + // delta_in = delta_out .* sigmoid'(x_in) + layer->sigmoid(x_in, &temp_result); + layer->d_sigmoid(&temp_result, &temp_result); + layer->multiply(&temp_result, delta_out, delta_in); + + return; +} + +void ailayer_sigmoid_calc_result_shape(ailayer_t *self) +{ + /* Unused: Shape is already defined (Pointer) + + self->result.shape[0] = self->input_layer->result.shape[0]; + self->result.shape[1] = self->input_layer->result.shape[1]; + */ + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void ailayer_sigmoid_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)) +{ + return; +} +#endif + diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_sigmoid.h b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_sigmoid.h new file mode 100644 index 0000000..1683f96 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_sigmoid.h @@ -0,0 +1,182 @@ +/** + * \file basic/base/ailayer/ailayer_sigmoid.h + * \internal + * \date 20.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link ailayer layer \endlink implementation of the Sigmoid activation layer + * + * This is an "abstract" data-type independent implementation. To use the layer use one of the provided + * implementations for a specific hardware and data-type (for example from ailayer_sigmoid_default.h) or set + * the required math functions on your own. + * + * \image html ailayer_sigmoid_schematic.png width=200px + * + * The Sigmoid layer is used as an activation function layer right after a dense layer. It calculates + * @f[ + * y = \sigma(x) = \frac{1}{1+e^{-x}} + * @f] + * for every element of the input tensor. + * + * The results of the forward pass of this layer are written to the result tensor of the base ailayer_t struct. + */ + +#ifndef SIGMOID_LAYER +#define SIGMOID_LAYER + +#include "core/aifes_core.h" + +typedef struct ailayer_sigmoid ailayer_sigmoid_t; + +/** @brief General \link ailayer_sigmoid.h Sigmoid layer \endlink struct +* +*/ +struct ailayer_sigmoid { + ailayer_t base; /**< Inherited field members from general ailayer struct. */ + const aimath_dtype_t *dtype; /**< Data type of the input and inference result values. */ + + /** @name Math functions + * @brief Required data type specific math functions + */ + ///@{ + + /** @brief Required math function: Sigmoid + * + * Requires a math function that calculates the element wise sigmoid of a tensor:\n + * @f[ + * result_{i} = \sigma(x_{i}) = \frac{1}{1 + e^{-x_{i}}} + * @f] + * + * @param x N-dimensional tensor (input) + * @param result N-dimensional tensor (output) + */ + void (*sigmoid)(const aitensor_t *x, aitensor_t *result); + + /** @brief Required math function: Derivative of sigmoid + * + * Requires a math function that calculates the element wise sigmoid derivative of a tensor:\n + * @f[ + * result_{i} = \sigma'(x_{i}) = \sigma(x_{i}) \cdot (1 - \sigma(x_{i})) + * @f] + * + * @param sigmoid_x N-dimensional tensor with the sigmoid values \f$ \sigma(x_{i}) \f$ (input) + * @param result N-dimensional tensor (output) + */ + void (*d_sigmoid)(const aitensor_t *sigmoid_x, aitensor_t *result); + + /** @brief Required math function: Element wise tensor multiplication + * + * Requires a math function that multiplies two tensors element wise:\n + * @f[ + * result = a \circ b + * @f] + */ + void (*multiply)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + ///@} +}; + +/** @brief Sigmoid layer type + * + * Defines the type of the layer (for example for type checks and debug prints). + * See aicore_layertype for more information about the layer type. + */ +extern const aicore_layertype_t *ailayer_sigmoid_type; + +/** @brief Initialize and connect the given Sigmoid layer + * + * This function represents the "constructor" of the abstract Sigmoid layer. It initializes the layer structure + * and connects it to the previous layer.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example ailayer_sigmoid_f32_default()). + * + * @param *layer The layer to initialize. + * @param *input_layer The previous layer that provides the inputs to the layer. + * @return Pointer to the (successfully) initialized general layer structure (ailayer_sigmoid.base). + */ +ailayer_t *ailayer_sigmoid(ailayer_sigmoid_t *layer, ailayer_t *input_layer); + +/** @brief Calculate the forward pass for given Sigmoid layer + * + * *Implementation of ailayer.forward.* + * + * It uses the result tensor of the previous layer as input and writes the result of the forward pass + * to the result tensor (ailayer.result) of the given layer. + * + * Calculation of the forward pass result: + * @f[ + * x_{out} \leftarrow \sigma(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ x_{out} \f$: Result of the forward pass of this layer\n\n + * + * Used math functions: + * * ailayer_sigmoid.sigmoid + * + * @param *self Layer to calculate the forward path for. + */ +void ailayer_sigmoid_forward(ailayer_t *self); + +/** @brief Calculate the backward pass for the given Sigmoid layer + * + * *Implementation of ailayer.backward.* + * + * It uses the deltas tensor of the next layer as input and writes the result of the backward pass + * to the deltas tensor (ailayer.deltas) of the given layer. + * + * Calculation of the errors for the previous layer: + * @f[ + * \delta_{in} \leftarrow \delta_{out} \circ \sigma'(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ \delta_{in} \f$: Result of the backward pass of this layer\n + * \f$ \delta_{out} \f$: Result of the backward pass of the next layer\n\n + * + * Used math functions: + * * ailayer_sigmoid.sigmoid + * * ailayer_sigmoid.d_sigmoid + * * ailayer_sigmoid.multiply + * + * @param *self Layer to calculate the backward path for. + */ +void ailayer_sigmoid_backward(ailayer_t *self); + +/** @brief Calculate the shape of the result tensor + * + * *Implementation of ailayer.calc_result_shape.* + * + * As the result tensor shape is shared with the result tensor shape of the previous layer (no change in shape is needed), + * this function returns without doing anything. + * + * @param *self Layer to calculate the resulting shape for. + */ +void ailayer_sigmoid_calc_result_shape(ailayer_t *self); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the layer specification + * + * @param *self The layer to print the specification for + * @param *print Pointer to the print function to use + */ +void ailayer_sigmoid_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // SIGMOID_LAYER diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_softmax.c b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_softmax.c new file mode 100644 index 0000000..f7d2ae4 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_softmax.c @@ -0,0 +1,95 @@ +/** + * \file basic/base/ailayer/ailayer_softmax.c + * \version 2.0alpha + * \date 07.12.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/ailayer/ailayer_softmax.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_layertype_t ailayer_softmax_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "Softmax", + .print_specs = ailayer_softmax_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_layertype_t *ailayer_softmax_type = &ailayer_softmax_type_s; + + +ailayer_t *ailayer_softmax(ailayer_softmax_t *layer, ailayer_t *input_layer) +{ + layer->base.layer_type = ailayer_softmax_type; + + layer->base.input_layer = input_layer; + input_layer->output_layer = &(layer->base); + + layer->base.layer_configuration = layer; + layer->base.result.dtype = layer->dtype; + layer->base.result.shape = input_layer->result.shape; + layer->base.result.dim = input_layer->result.dim; + + // No backward pass supported yet + layer->base.deltas.dtype = 0; + + layer->base.forward = ailayer_softmax_forward; + layer->base.backward = 0; + + layer->base.calc_result_shape = ailayer_softmax_calc_result_shape; + layer->base.sizeof_paramem = 0; + layer->base.set_paramem = 0; + layer->base.sizeof_trainmem = 0; + layer->base.set_trainmem = 0; + + layer->base.trainable_params_count = 0; + + return &(layer->base); +} + +void ailayer_softmax_forward(ailayer_t *self) +{ + ailayer_softmax_t *layer = (ailayer_softmax_t *)(self->layer_configuration); + aitensor_t *x_in = &(self->input_layer->result); + aitensor_t *x_out = &(self->result); + + layer->softmax(x_in, x_out); + return; +} + +void ailayer_softmax_calc_result_shape(ailayer_t *self) +{ + /* Unused: Shape is already defined (Pointer) + + self->result.shape[0] = self->input_layer->result.shape[0]; + self->result.shape[1] = self->input_layer->result.shape[1]; + */ + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void ailayer_softmax_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)) +{ + return; +} +#endif + diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_softmax.h b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_softmax.h new file mode 100644 index 0000000..9b97ec2 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_softmax.h @@ -0,0 +1,140 @@ + +/** + * \file basic/base/ailayer/ailayer_softmax.h + * \internal + * \date 07.12.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link ailayer layer \endlink implementation of the Softmax activation layer + * + * This is an "abstract" data-type independent implementation. To use the layer use one of the provided + * implementations for a specific hardware and data-type (for example from ailayer_softmax_default.h) or set + * the required math functions on your own.\n + * This layer is designed as an output-layer and has no backward implementation. If you want to train an ANN with this layer, + * use the \link ailoss_crossentropy.h Cross-Entropy loss \endlink which has a combined Softmax-Cross-Entropy backward + * function that is faster to compute! + * + * \image html ailayer_softmax_schematic.png width=200px + * + * The Softmax layer is used as an activation function layer right after a dense layer. It calculates + * @f[ + * y_i = \frac{e^{x_i}}{\sum_{j=1}^{K} e^{x_j}} + * @f] + * for every element of the input tensor with \f$ K \f$ elements. + * + * The results of the forward pass of this layer are written to the result tensor of the base ailayer_t struct. + */ + +#ifndef SOFTMAX_LAYER +#define SOFTMAX_LAYER + +#include "core/aifes_core.h" + +typedef struct ailayer_softmax ailayer_softmax_t; + +/** @brief General \link ailayer_softmax.h Softmax layer \endlink struct +* +*/ +struct ailayer_softmax { + ailayer_t base; /**< Inherited field members from general ailayer struct. */ + const aimath_dtype_t *dtype; /**< Data type of the input and inference result values. */ + + /** @name Math functions + * @brief Required data type specific math functions + */ + ///@{ + + /** @brief Required math function: Softmax + * + * Requires a math function that calculates the element wise softmax of a tensor:\n + * @f[ + * result_{i} = \frac{e^{x_i}}{\sum_{j=1}^{K} e^{x_j}} + * @f] + * + * @param x N-dimensional tensor (input) + * @param result N-dimensional tensor (output) + */ + void (*softmax)(const aitensor_t *x, aitensor_t *result); + + ///@} +}; + +/** @brief Softmax layer type + * + * Defines the type of the layer (for example for type checks and debug prints). + * See aicore_layertype for more information about the layer type. + */ +extern const aicore_layertype_t *ailayer_softmax_type; + +/** @brief Initialize and connect the given Softmax layer + * + * This function represents the "constructor" of the abstract Softmax layer. It initializes the layer structure + * and connects it to the previous layer.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example ailayer_softmax_f32_default()). + * + * @param *layer The layer to initialize. + * @param *input_layer The previous layer that provides the inputs to the layer. + * @return Pointer to the (successfully) initialized general layer structure (ailayer_softmax.base). + */ +ailayer_t *ailayer_softmax(ailayer_softmax_t *layer, ailayer_t *input_layer); + +/** @brief Calculate the forward pass for given Softmax layer + * + * *Implementation of ailayer.forward.* + * + * It uses the result tensor of the previous layer as input and writes the result of the forward pass + * to the result tensor (ailayer.result) of the given layer. + * + * Calculation of the forward pass result: + * @f[ + * x_{out} \leftarrow Softmax(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ x_{out} \f$: Result of the forward pass of this layer\n\n + * + * Used math functions: + * * ailayer_softmax.softmax + * + * @param *self Layer to calculate the forward path for. + */ +void ailayer_softmax_forward(ailayer_t *self); + +/** @brief Calculate the shape of the result tensor + * + * *Implementation of ailayer.calc_result_shape.* + * + * As the result tensor shape is shared with the result tensor shape of the previous layer (no change in shape is needed), + * this function returns without doing anything. + * + * @param *self Layer to calculate the resulting shape for. + */ +void ailayer_softmax_calc_result_shape(ailayer_t *self); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the layer specification + * + * @param *self The layer to print the specification for + * @param *print Pointer to the print function to use + */ +void ailayer_softmax_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // SOFTMAX_LAYER diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_softsign.c b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_softsign.c new file mode 100644 index 0000000..9cc1152 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_softsign.c @@ -0,0 +1,120 @@ +/** + * \file basic/base/ailayer/ailayer_softsign.c + * \version 2.0alpha + * \date 16.03.2021 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief A softsign activation function. + * \details + */ + +#include "basic/base/ailayer/ailayer_softsign.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_layertype_t ailayer_softsign_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "Softsign", + .print_specs = ailayer_softsign_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_layertype_t *ailayer_softsign_type = &ailayer_softsign_type_s; + + +ailayer_t *ailayer_softsign(ailayer_softsign_t *layer, ailayer_t *input_layer) +{ + layer->base.layer_type = ailayer_softsign_type; + + layer->base.input_layer = input_layer; + input_layer->output_layer = &(layer->base); + + layer->base.layer_configuration = layer; + layer->base.result.dtype = layer->dtype; + + layer->base.result.dim = input_layer->result.dim; + layer->base.result.shape = input_layer->result.shape; + + layer->base.deltas.dtype = layer->dtype; + layer->base.deltas.dim = 2; + layer->base.deltas.shape = layer->base.result.shape; + + layer->base.forward = ailayer_softsign_forward; + layer->base.backward = ailayer_softsign_backward; + + layer->base.calc_result_shape = ailayer_softsign_calc_result_shape; + layer->base.sizeof_paramem = 0; + layer->base.set_paramem = 0; + layer->base.sizeof_trainmem = 0; + layer->base.set_trainmem = 0; + + layer->base.trainable_params_count = 0; + + return &(layer->base); +} + +void ailayer_softsign_backward(ailayer_t *self) +{ + ailayer_softsign_t *layer = (ailayer_softsign_t *)(self->layer_configuration); + aitensor_t *delta_in = &(self->deltas); + aitensor_t *delta_out = &(self->output_layer->deltas); + aitensor_t *x_in = &(self->input_layer->result); + + int8_t temp_result_params[aimath_sizeof_tensor_params(x_in) + 1]; // +1 to prevent array of size 0 + int8_t temp_result_data[aimath_sizeof_tensor_data(x_in)]; + aitensor_t temp_result = { + .dim = 2, + .shape = x_in->shape, + .data = temp_result_data, + .dtype = x_in->dtype, + .tensor_params = temp_result_params + }; + + layer->softsign(x_in, &temp_result); + layer->d_softsign(&temp_result, &temp_result); + layer->multiply(&temp_result, delta_out, delta_in); + + return; +} + +void ailayer_softsign_forward(ailayer_t *self) +{ + ailayer_softsign_t *layer = (ailayer_softsign_t *)(self->layer_configuration); + aitensor_t *x_in = &(self->input_layer->result); + aitensor_t *x_out = &(self->result); + + layer->softsign(x_in, x_out); + return; +} + +void ailayer_softsign_calc_result_shape(ailayer_t *self) +{ + /* Unused: Shape is already defined (Pointer) + + self->result.shape[0] = self->input_layer->result.shape[0]; + self->result.shape[1] = self->input_layer->result.shape[1]; + */ + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void ailayer_softsign_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)) +{ + return; +} +#endif diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_softsign.h b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_softsign.h new file mode 100644 index 0000000..01524b8 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_softsign.h @@ -0,0 +1,183 @@ + +/** + * \file basic/base/ailayer/ailayer_softsign.h + * \internal + * \date 16.03.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link ailayer layer \endlink implementation of the Softsign activation layer + * + * This is an "abstract" data-type independent implementation. To use the layer use one of the provided + * implementations for a specific hardware and data-type (for example from ailayer_softsign_default.h) or set + * the required math functions on your own. + * + * \image html ailayer_softsign_schematic.png width=200px + * + * The Softsign layer is used as an activation function layer right after a dense layer. It calculates + * @f[ + * y = \frac {x} {1 + |x|} + * @f] + * for every element of the input tensor. + * + * The results of the forward pass of this layer are written to the result tensor of the base ailayer_t struct. + */ + +#ifndef SOFTSIGN_LAYER +#define SOFTSIGN_LAYER + +#include "core/aifes_core.h" + +typedef struct ailayer_softsign ailayer_softsign_t; + +/** @brief General \link ailayer_softsign.h Softsign layer \endlink struct +* +*/ +struct ailayer_softsign { + ailayer_t base; /**< Inherited field members from general ailayer struct. */ + const aimath_dtype_t *dtype; /**< Data type of the input and inference result values. */ + + /** @name Math functions + * @brief Required data type specific math functions + */ + ///@{ + + /** @brief Required math function: Softsign + * + * Requires a math function that calculates the element wise softsign of a tensor:\n + * @f[ + * result_{i} = \frac {x_i} {1 + |x_i|} + * @f] + * + * @param x N-dimensional tensor (input) + * @param result N-dimensional tensor (output) + */ + void (*softsign)(const aitensor_t *x, aitensor_t *result); + + /** @brief Required math function: Derivative of softsign + * + * Requires a math function that calculates the element wise softsign derivative of a tensor:\n + * @f[ + * result_{i} = \frac {x_i} {(1 + |x_i|)^2} + * @f] + * + * @param x N-dimensional tensor (input) + * @param result N-dimensional tensor (output) + */ + void (*d_softsign)(const aitensor_t *softsign_x, aitensor_t *result); + + /** @brief Required math function: Element wise tensor multiplication + * + * Requires a math function that multiplies two tensors element wise:\n + * @f[ + * result = a \circ b + * @f] + */ + void (*multiply)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + ///@} +}; + +/** @brief Softsign layer type + * + * Defines the type of the layer (for example for type checks and debug prints). + * See aicore_layertype for more information about the layer type. + */ +extern const aicore_layertype_t *ailayer_softsign_type; + +/** @brief Initialize and connect the given Softsign layer + * + * This function represents the "constructor" of the abstract Softsign layer. It initializes the layer structure + * and connects it to the previous layer.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example ailayer_softsign_f32_default()). + * + * @param *layer The layer to initialize. + * @param *input_layer The previous layer that provides the inputs to the layer. + * @return Pointer to the (successfully) initialized general layer structure (ailayer_softsign.base). + */ +ailayer_t *ailayer_softsign(ailayer_softsign_t *layer, ailayer_t *input_layer); + +/** @brief Calculate the forward pass for given Tanh layer + * + * *Implementation of ailayer.forward.* + * + * It uses the result tensor of the previous layer as input and writes the result of the forward pass + * to the result tensor (ailayer.result) of the given layer. + * + * Calculation of the forward pass result: + * @f[ + * x_{out} \leftarrow Softsign(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ x_{out} \f$: Result of the forward pass of this layer\n\n + * + * Used math functions: + * * ailayer_softsign.softsign + * + * @param *self Layer to calculate the forward pass for. + */ +void ailayer_softsign_forward(ailayer_t *self); + +/** @brief Calculate the backward pass for the given Softsign layer + * + * *Implementation of ailayer.backward.* + * + * It uses the deltas tensor of the next layer as input and writes the result of the backward pass + * to the deltas tensor (ailayer.deltas) of the given layer. + * + * Calculation of the errors for the previous layer: + * @f[ + * \delta_{in} \leftarrow \delta_{out} \circ Softsign'(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ \delta_{in} \f$: Result of the backward pass of this layer\n + * \f$ \delta_{out} \f$: Result of the backward pass of the next layer\n\n + * + * Used math functions: + * * ailayer_softsign.softsign + * * ailayer_softsign.d_softsign + * * ailayer_softsign.multiply + * + * @param *self Layer to calculate the backward path for. + */ +void ailayer_softsign_backward(ailayer_t *self); + +/** @brief Calculate the shape of the result tensor + * + * *Implementation of ailayer.calc_result_shape.* + * + * As the result tensor shape is shared with the result tensor shape of the previous layer (no change in shape is needed), + * this function returns without doing anything. + * + * @param *self Layer to calculate the resulting shape for. + */ +void ailayer_softsign_calc_result_shape(ailayer_t *self); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the layer specification + * + * @param *self The layer to print the specification for + * @param *print Pointer to the print function to use + */ +void ailayer_softsign_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // SOFTSIGN_LAYER diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_tanh.c b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_tanh.c new file mode 100644 index 0000000..ed88e2b --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_tanh.c @@ -0,0 +1,123 @@ + +/** + * \file basic/base/ailayer/ailayer_tanh.c + * \version 2.0alpha + * \date 17.03.2021 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/ailayer/ailayer_tanh.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_layertype_t ailayer_tanh_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "Tanh", + .print_specs = ailayer_tanh_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_layertype_t *ailayer_tanh_type = &ailayer_tanh_type_s; + + +ailayer_t *ailayer_tanh(ailayer_tanh_t *layer, ailayer_t *input_layer) +{ + layer->base.layer_type = ailayer_tanh_type; + + layer->base.input_layer = input_layer; + input_layer->output_layer = &(layer->base); + + layer->base.layer_configuration = layer; + layer->base.result.dtype = layer->dtype; + layer->base.result.dim = input_layer->result.dim; + layer->base.result.shape = input_layer->result.shape; + + layer->base.deltas.dtype = layer->dtype; + layer->base.deltas.dim = 2; + layer->base.deltas.shape = layer->base.result.shape; + + layer->base.forward = ailayer_tanh_forward; + layer->base.backward = ailayer_tanh_backward; + + layer->base.calc_result_shape = ailayer_tanh_calc_result_shape; + layer->base.sizeof_paramem = 0; + layer->base.set_paramem = 0; + layer->base.sizeof_trainmem = 0; + layer->base.set_trainmem = 0; + + layer->base.trainable_params_count = 0; + + return &(layer->base); +} + +void ailayer_tanh_forward(ailayer_t *self) +{ + ailayer_tanh_t *layer = (ailayer_tanh_t *)(self->layer_configuration); + aitensor_t *x_in = &(self->input_layer->result); + aitensor_t *x_out = &(self->result); + + layer->tanh(x_in, x_out); + return; +} + + +void ailayer_tanh_backward(ailayer_t *self) +{ + ailayer_tanh_t *layer = (ailayer_tanh_t *)(self->layer_configuration); + aitensor_t *delta_in = &(self->deltas); + aitensor_t *delta_out = &(self->output_layer->deltas); + aitensor_t *x_in = &(self->input_layer->result); + + int8_t temp_result_params[aimath_sizeof_tensor_params(x_in) + 1]; // +1 to prevent array of size 0 + int8_t temp_result_data[aimath_sizeof_tensor_data(x_in)]; + aitensor_t temp_result = { + .dim = 2, + .shape = x_in->shape, + .data = temp_result_data, + .dtype = x_in->dtype, + .tensor_params = temp_result_params + }; + + // delta_in = delta_out .* tanh'(x_in) + layer->tanh(x_in, &temp_result); + layer->d_tanh(&temp_result, &temp_result); + layer->multiply(&temp_result, delta_out, delta_in); + + return; +} + +void ailayer_tanh_calc_result_shape(ailayer_t *self) +{ + /* Unused: Shape is already defined (Pointer) + + self->result.shape[0] = self->input_layer->result.shape[0]; + self->result.shape[1] = self->input_layer->result.shape[1]; + */ + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void ailayer_tanh_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)) +{ + return; +} +#endif + diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_tanh.h b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_tanh.h new file mode 100644 index 0000000..177aef5 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_tanh.h @@ -0,0 +1,182 @@ + +/** + * \file basic/base/ailayer/ailayer_tanh.h + * \internal + * \date 17.03.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link ailayer layer \endlink implementation of the Tanh activation layer + * + * This is an "abstract" data-type independent implementation. To use the layer use one of the provided + * implementations for a specific hardware and data-type (for example from ailayer_tanh_default.h) or set + * the required math functions on your own. + * + * \image html ailayer_tanh_schematic.png width=200px + * + * The Tanh layer is used as an activation function layer right after a dense layer. It calculates + * @f[ + * y = \tanh(x) = \frac{e^x - e^{-x}}{e^x + e^{-x}} + * @f] + * for every element of the input tensor. + * + * The results of the forward pass of this layer are written to the result tensor of the base ailayer_t struct. + */ +#ifndef TANH_LAYER +#define TANH_LAYER + +#include "core/aifes_core.h" + +typedef struct ailayer_tanh ailayer_tanh_t; + +/** @brief General \link ailayer_tanh.h Tanh layer \endlink struct +* +*/ +struct ailayer_tanh { + ailayer_t base; /**< Inherited field members from general ailayer struct. */ + const aimath_dtype_t *dtype; /**< Data type of the input and inference result values. */ + + /** @name Math functions + * @brief Required data type specific math functions + */ + ///@{ + + /** @brief Required math function: Tanh + * + * Requires a math function that calculates the element wise tanh of a tensor:\n + * @f[ + * result_{i} = \tanh(x_{i}) = \frac{e^{x_i} - e^{-x_i}}{e^{x_i} + e^{-x_i}} + * @f] + * + * @param x N-dimensional tensor (input) + * @param result N-dimensional tensor (output) + */ + void (*tanh)(const aitensor_t *x, aitensor_t *result); + + /** @brief Required math function: Derivative of tanh + * + * Requires a math function that calculates the element wise tanh derivative of a tensor:\n + * @f[ + * result_{i} = tanh'(x_{i}) = 1 - tanh(x_{i})^2 + * @f] + * + * @param tanh_x N-dimensional tensor with the tanh values \f$ tanh(x_{i}) \f$ (input) + * @param result N-dimensional tensor (output) + */ + void (*d_tanh)(const aitensor_t *tanh_x, aitensor_t *result); + + /** @brief Required math function: Element wise tensor multiplication + * + * Requires a math function that multiplies two tensors element wise:\n + * @f[ + * result = a \circ b + * @f] + */ + void (*multiply)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + ///@} +}; + +/** @brief Tanh layer type + * + * Defines the type of the layer (for example for type checks and debug prints). + * See aicore_layertype for more information about the layer type. + */ +extern const aicore_layertype_t *ailayer_tanh_type; + +/** @brief Initialize and connect the given Tanh layer + * + * This function represents the "constructor" of the abstract Tanh layer. It initializes the layer structure + * and connects it to the previous layer.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example ailayer_tanh_f32_default()). + * + * @param *layer The layer to initialize. + * @param *input_layer The previous layer that provides the inputs to the layer. + * @return Pointer to the (successfully) initialized general layer structure (ailayer_tanh.base). + */ +ailayer_t *ailayer_tanh(ailayer_tanh_t *layer, ailayer_t *input_layer); + +/** @brief Calculate the forward pass for given Tanh layer + * + * *Implementation of ailayer.forward.* + * + * It uses the result tensor of the previous layer as input and writes the result of the forward pass + * to the result tensor (ailayer.result) of the given layer. + * + * Calculation of the forward pass result: + * @f[ + * x_{out} \leftarrow tanh(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ x_{out} \f$: Result of the forward pass of this layer\n\n + * + * Used math functions: + * * ailayer_tanh.tanh + * + * @param *self Layer to calculate the forward path for. + */ +void ailayer_tanh_forward(ailayer_t *self); + +/** @brief Calculate the backward pass for the given Tanh layer + * + * *Implementation of ailayer.backward.* + * + * It uses the deltas tensor of the next layer as input and writes the result of the backward pass + * to the deltas tensor (ailayer.deltas) of the given layer. + * + * Calculation of the errors for the previous layer: + * @f[ + * \delta_{in} \leftarrow \delta_{out} \circ tanh'(x_{in}) + * @f] + * + * \f$ x_{in} \f$: Result of the forward pass of the previous layer\n + * \f$ \delta_{in} \f$: Result of the backward pass of this layer\n + * \f$ \delta_{out} \f$: Result of the backward pass of the next layer\n\n + * + * Used math functions: + * * ailayer_tanh.tanh + * * ailayer_tanh.d_tanh + * * ailayer_tanh.multiply + * + * @param *self Layer to calculate the backward path for. + */ +void ailayer_tanh_backward(ailayer_t *self); + +/** @brief Calculate the shape of the result tensor + * + * *Implementation of ailayer.calc_result_shape.* + * + * As the result tensor shape is shared with the result tensor shape of the previous layer (no change in shape is needed), + * this function returns without doing anything. + * + * @param *self Layer to calculate the resulting shape for. + */ +void ailayer_tanh_calc_result_shape(ailayer_t *self); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the layer specification + * + * @param *self The layer to print the specification for + * @param *print Pointer to the print function to use + */ +void ailayer_tanh_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // TANH_LAYER diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_template.c b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_template.c new file mode 100644 index 0000000..2c39b9b --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_template.c @@ -0,0 +1,211 @@ +/** + * \file basic/base/ailayer/ailayer_template.c + * \version 2.0alpha + * \date 20.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/ailayer/ailayer_template.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_layertype_t ailayer_template_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "Template", + .print_specs = ailayer_template_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_layertype_t *ailayer_template_type = &ailayer_template_type_s; + +ailayer_t *ailayer_template(ailayer_template_t *layer, ailayer_t *input_layer) +{ + // Connect the layer with its input + layer->base.input_layer = input_layer; + input_layer->output_layer = &(layer->base); + + // Init basic layer variables + layer->base.layer_configuration = layer; + layer->base.result.dtype = layer->dtype; + layer->base.result.shape = layer->result_shape; + layer->base.result.dim = 2; + + layer->base.deltas.dtype = layer->dtype; + layer->base.deltas.shape = input_layer->result.shape; + layer->base.deltas.dim = 2; + + // Set the function pointers + layer->base.forward = ailayer_template_forward; + layer->base.backward = ailayer_template_backward; + + layer->base.calc_result_shape = ailayer_template_calc_result_shape; + layer->base.sizeof_paramem = ailayer_template_sizeof_paramem; + layer->base.set_paramem = ailayer_template_set_paramem; + layer->base.sizeof_paramem = ailayer_template_sizeof_trainmem; + layer->base.set_trainmem = ailayer_template_set_trainmem; + + return &layer->base; +} + +/* +* Layer naming: +* ------- +* x_in -> | L | -> x_out +* | A | +* | Y | +* | E | +* delta_in <- | R | <- delta_out +* ------- +*/ + + +void ailayer_template_forward(ailayer_t *self) +{ + ailayer_template_t *layer = (ailayer_template_t *)(self->layer_configuration); + aitensor_t *x_in = &(self->input_layer->result); + aitensor_t *x_out = &(self->result); + + // Example forward: x_out = x_in + params + layer->tensor_add(x_in, layer->params, x_out); + + return; +} + +void ailayer_template_backward(ailayer_t *self) +{ + ailayer_template_t *layer = (ailayer_template_t *)(self->layer_configuration); + aitensor_t *x_in = &(self->input_layer->result); +// aitensor_t *x_out = &(self->result); + aitensor_t *delta_in = &(self->deltas); + aitensor_t *delta_out = &(self->output_layer->deltas); + + // Temporary result for calculation can be created here. Document the needed amount of temp memory! + // Remove code if unused! + float temp_result_data[delta_out->shape[0] * delta_out->shape[1]]; + uint16_t temp_result_shape[] = {delta_out->shape[0], delta_out->shape[1]}; + aitensor_t temp_result = { + .dim = 2, + .shape = temp_result_shape, + .data = temp_result_data, + .dtype = x_in->dtype + }; + + // 1) Calculate the gradients and add to the gradients tensor + layer->copy_tensor(delta_out, &temp_result); // (Unnecessary, just show the use of temp_result) + layer->tensor_add(layer->d_params, &temp_result, layer->d_params); + + // 2) Calculate delta for next layer. Do not before calculating gradients!!! May override x_in!!! + layer->copy_tensor(&temp_result, delta_in); + + return; +} + +void ailayer_template_calc_result_shape(ailayer_t *self) +{ +// ailayer_template_t *layer = (ailayer_template_t *)(self->layer_configuration); + aitensor_t *x_in = &(self->input_layer->result); + aitensor_t *x_out = &(self->result); + + x_out->shape[0] = x_in->shape[0]; + x_out->shape[1] = x_in->shape[1]; + + return; +} + +uint32_t ailayer_template_sizeof_paramem(const ailayer_t *self) +{ + ailayer_template_t *layer = (ailayer_template_t *)(self->layer_configuration); +// const aitensor_t *x_in = &(self->input_layer->result); + const aitensor_t *x_out = &(self->result); + + uint32_t memory = 0; + + // Memory amount for params. Attention: result shape is calculated but params tensor is not available yet. + memory += sizeof(aitensor_t); // struct + memory += 2 * sizeof(uint16_t); // shape array + memory += x_out->shape[0] * x_out->shape[1] * aimath_sizeof_dtype(layer->dtype); // data + + return memory; +} + +void ailayer_template_set_paramem(ailayer_t *self, void *memory_ptr) +{ + ailayer_template_t *layer = (ailayer_template_t *)(self->layer_configuration); + aitensor_t *x_in = &(self->input_layer->result); +// aitensor_t *x_out = &(self->result); + + uint32_t address_counter = 0; + + // Params memory distribution + // tensor struct: + layer->params = memory_ptr + address_counter; + address_counter += sizeof(aitensor_t); + layer->params->dim = 2; + layer->params->dtype = layer->dtype; + // shape array: + layer->params->shape = memory_ptr + address_counter; + address_counter += 2 * sizeof(uint16_t); + layer->params->shape[0] = x_in->shape[0]; + layer->params->shape[1] = x_in->shape[1]; + // data: + layer->params->data = memory_ptr + address_counter; + address_counter += aimath_sizeof_tensor_data(layer->params); + + return; +} + +uint32_t ailayer_template_sizeof_trainmem(const ailayer_t *self) +{ + ailayer_template_t *layer = (ailayer_template_t *)(self->layer_configuration); + + uint32_t memory = 0; + + // Sum up the memory sizes for every parameter tensor + memory += aimath_sizeof_tensor(layer->params); // Size of tensor struct and data but not the shape array, because shape array is shared with params + + return memory; +} + +void ailayer_template_set_trainmem(ailayer_t *self, void *memory_ptr) +{ + ailayer_template_t *layer = (ailayer_template_t *)(self->layer_configuration); + + uint32_t address_counter = 0; + + // Params gradient memory distribution + // tensor struct: + layer->d_params = memory_ptr + address_counter; + address_counter += sizeof(aitensor_t); + layer->d_params->dim = 2; + layer->d_params->shape = layer->params->shape; // shared shape + // data: + layer->d_params->data = memory_ptr + address_counter; + address_counter += aimath_sizeof_tensor_data(layer->d_params); + + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void ailayer_template_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)) +{ + return; +} +#endif diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_template.h b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_template.h new file mode 100644 index 0000000..c5f28d4 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailayer/ailayer_template.h @@ -0,0 +1,131 @@ +/** + * \file basic/base/ailayer/ailayer_template.h + * \internal + * \date 20.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief [DEPRECATED] This file acts as a reference for new layer structures. + * Some of the shown functions are mandatory, some are optional. + * \details + */ + +#ifndef TEMPLATE_LAYER +#define TEMPLATE_LAYER + +#include "core/aifes_core.h" + +typedef struct ailayer_template ailayer_template_t; + +/** @brief Template for general AIfES layers +* +*/ +struct ailayer_template { + ailayer_t base; /**< Inherited field members from general ailayer struct. */ // The base or super "class" of the layer. + const aimath_dtype_t *dtype; /**< Datatype of the inference result values. */ + + // Configurations of the layer, that can be configured by the user when creating the layer + uint32_t example_configuration; + + // Tensor pointer for trainable parameters [DEPRECATED] + aitensor_t *params; + aitensor_t *d_params; // Gradients + + // If the shape of the result differs from the input shape, this array can be used for the result. + // If not, the result shape array of the input tensor can be used for the result tensor too. + uint16_t result_shape[2]; /**< Inference result tensor shape (n x m matrix). */ + + aitensor_t *trainable_params[2]; /**< Pointer to the weights and biases (which are the trainable parameters). */ + aitensor_t *gradients[2]; /**< Gradients structure for the back propagation algorithm. */ + void *optimem[2]; /**< Memory field used by the trainings optimizer. */ + + // Make the math functions accessable for expert developers + void (*copy_tensor)(const aitensor_t *from, aitensor_t *to); /**< Function pointer to the used tensor copy function. */ + void (*tensor_add)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); /**< Function pointer to the used tensor addition. */ +}; + +extern const aicore_layertype_t *ailayer_template_type; +/** @brief Initialize the given layer struct - __mandatory__ + * + * Initializes and sets all parameters needed by the layer structure. + * + * @param *layer The layer to initialize. + * @param *input_layer The layer that provides the inputs to the initialized layer. + * @return Pointer to the (successfully) initialized layer structure. + */ + +ailayer_t *ailayer_template(ailayer_template_t *layer, ailayer_t *input_layer); + +/** @brief Calculate the forward path for given layer - __mandatory__ + * + * @param *self Layer to calculate the forward path for. + */ +void ailayer_template_forward(ailayer_t *self); + +/** @brief Calculate the backward (training) path for the given layer - __depends on usability in training__ + * + * @param *self Layer to calculate the backward path for. + */ +void ailayer_template_backward(ailayer_t *self); + +/** @brief Calculate the shape of the result tensor + * + * @param *self Layer to calculate the resulting shape for. + */ +void ailayer_template_calc_result_shape(ailayer_t *self); + +/** @brief Calculate and return the parameter memory size + * needed for this layer - __mandatory for hidden layers__ + * + * @param *self The layer to calculate the parameter memory size for + * @return Calculated parameter memory size in bytes. + */ +uint32_t ailayer_template_sizeof_paramem(const ailayer_t *self); + +/** @brief Distribute provided memory to the parameter pointers - __mandatory for hidden layers__ + * + * @param *self The layer to set the memory fields for. + * @param *memory_ptr Pointer to the buffer provided for the layer parameters. + */ +void ailayer_template_set_paramem(ailayer_t *self, void *memory_ptr); + +/** @brief Calculate and return the necessary memory size + * needed by this layer for training - __mandatory for hidden layers__ + * + * @param *self The layer to calculate the gradient memory size for. + * @return Calculated gradient memory size in bytes. + */ +uint32_t ailayer_template_sizeof_trainmem(const ailayer_t *self); + +/** @brief Distribute provided memory to the training gradients pointers - __mandatory for hidden layers__ + * + * @param *self The layer to set the memory fields for. + * @param *memory_ptr Pointer to the buffer provided for the layer gradients. + */ +void ailayer_template_set_trainmem(ailayer_t *self, void *memory_ptr); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the layer specification + * + * @param *self The layer to print the specification for + * @param *print Pointer to the print function to use + */ +void ailayer_template_print_specs(const ailayer_t *self, int (*print)(const char *format, ...)); +#endif + +#endif // TEMPLATE_LAYER diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailoss/ailoss_crossentropy.c b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailoss/ailoss_crossentropy.c new file mode 100644 index 0000000..1591d70 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailoss/ailoss_crossentropy.c @@ -0,0 +1,110 @@ +/** + * \file basic/base/ailoss/ailoss_crossentropy.c + * \version 2.0alpha + * \date 12.01.2021 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/ailoss/ailoss_crossentropy.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_losstype_t ailoss_crossentropy_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "Cross-entropy", + .print_specs = ailoss_crossentropy_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_losstype_t *ailoss_crossentropy_type = &ailoss_crossentropy_type_s; + +ailoss_t *ailoss_crossentropy(ailoss_crossentropy_t *loss, ailayer_t *input_layer) +{ + loss->base.loss_type = ailoss_crossentropy_type; + + loss->base.connection_layer.input_layer = input_layer; + input_layer->output_layer = &(loss->base.connection_layer); + + loss->base.loss_configuration = loss; + + loss->base.connection_layer.deltas.dtype = loss->dtype; + + loss->base.calc_delta = ailoss_crossentropy_calc_delta; + loss->base.calc_loss = ailoss_crossentropy_calc_loss; + + // Check for valid input and override the backward function (not needed) + if(input_layer->layer_type == ailayer_softmax_type || input_layer->layer_type == ailayer_sigmoid_type){ + input_layer->backward = ailoss_crossentropy_dummy_backward; + } + else{ + #ifdef AIDEBUG_PRINT_ERROR_MESSAGES + printf("\n!!! ERROR !!! (ailoss_crossentropy): No valid input layer. Use either Sigmoid or Softmax as input.\n"); + #endif + return 0; + } + + return &loss->base; +} + +void ailoss_crossentropy_calc_delta(ailoss_t *self, const aitensor_t *target_data) +{ + ailoss_crossentropy_t *loss = (ailoss_crossentropy_t *)(self->loss_configuration); + aitensor_t *predicted_data = &(self->connection_layer.input_layer->result); + aitensor_t *deltas = &(self->connection_layer.input_layer->deltas); + + deltas->shape = predicted_data->shape; // ToDo: Maybe remove or put it in constructor + + // Calc dC/dz directly instead of dC/da_L and then da_L/dz in the output layer + // dC/dz = (a_L - y) ( * 1/batch_size) + loss->tensor_sub(predicted_data, target_data, deltas); + + // ToDo: Scale by batch size. (Divide / Shift) + + return; +} + +void ailoss_crossentropy_calc_loss(ailoss_t *self, const aitensor_t *target_data, void *result) +{ + ailoss_crossentropy_t *loss = (ailoss_crossentropy_t *)(self->loss_configuration); + + aitensor_t *predicted_data = &(self->connection_layer.input_layer->result); + + //C = - y * log(a_L) ( * 1/batch_size) + loss->crossentropy(predicted_data, target_data, result); + + // ToDo: Scale by batch size. (Divide / Shift) + + return; +} + +void ailoss_crossentropy_dummy_backward(ailayer_t *self) +{ + return; +} + + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void ailoss_crossentropy_print_specs(const ailoss_t *self, int (*print)(const char *format, ...)) +{ + return; +} +#endif + diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailoss/ailoss_crossentropy.h b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailoss/ailoss_crossentropy.h new file mode 100644 index 0000000..f8cb30e --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailoss/ailoss_crossentropy.h @@ -0,0 +1,205 @@ +/** + * \file basic/base/ailoss/ailoss_crossentropy.h + * \internal + * \date 12.01.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link ailoss loss \endlink implementation of the Cross-Entropy loss + * + * This is an "abstract" data-type independent implementation. To use the loss, use one of the provided + * implementations for a specific hardware and data-type (for example from ailoss_crossentropy_default.h) or set + * the required math functions on your own. + * + * The Cross-Entropy loss ist best suitable for classification tasks + * and works only with \link ailayer_sigmoid.h Sigmoid \endlink or \link ailayer_softmax.h Softmax \endlink output layers. + * + * For **binary classification** (binary targets 0 or 1) with **Sigmoid output layer**, the loss / cost is calculated as + * @f[ + * L(y, \hat{y}) = - \sum_{i=0}^{N} (y_i \log(\hat{y}_i) + (1-y) \log(1-\hat{y}_i)) + * @f] + * with the predicted values \f$ \hat{y}_i \f$ and the target values \f$ y_i \f$. + * \f$ N \f$ is the number of elements of the \f$ y \f$ tensor. + * + * For **categorigal classification** (one-hot encoded targets) with **Softmax output layer**, the loss / cost is calculated as + * @f[ + * L(y, \hat{y}) = - \sum_{i=0}^{N} y_{i} \log(\hat{y}_{i}) + * @f] + * with the predicted values \f$ \hat{y}_i \f$ and the target values \f$ y_i \f$. + * \f$ N \f$ is the number of elements of the \f$ y \f$ tensor. + * + * To get the "mean" normalization, you have to modify the learning rate to \f$ lr = \frac {1}{o \cdot n} \cdot lr \f$ + * with the number of outputs \f$ o \f$ and the batch size \f$ n \f$. + * + * The loss can be calculated with ailoss_crossentropy_calc_loss(). For training the deltas /errors on the target data are + * calculated with ailoss_crossentropy_calc_delta() and written to the deltas tensor of the connection layer. + */ + +#ifndef CROSSENTROPY_LOSS +#define CROSSENTROPY_LOSS + +#include "core/aifes_core.h" +#include "basic/base/ailayer/ailayer_softmax.h" +#include "basic/base/ailayer/ailayer_sigmoid.h" + +typedef struct ailoss_crossentropy ailoss_crossentropy_t; /**< New data type name for code reduction. */ + +/** @brief General \link ailoss_crossentropy.h Cross-Entropy loss \endlink struct +* +*/ +struct ailoss_crossentropy { + ailoss_t base; /**< Inherited field members from general ailoss struct. */ + const aimath_dtype_t *dtype; /**< Main data type of the loss. */ + + /** @name Math functions + * @brief Required data type specific math functions + */ + ///@{ + + /** @brief Required math function: Element wise tensor subtraction + * + * Requires a math function that subtracts two tensors element wise:\n + * @f[ + * result = a - b + * @f] + */ + void (*tensor_sub)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + /** @brief Required math function: Cross-Entropy between two tensors + * + * Requires a math function that calculates the Cross-Entropy between two tensors.\n + * For a Sigmoid output layer it should be the binary Cross-Entropy: + * @f[ + * result = -\sum (target \cdot \log(predicted) + (1 - target) \log(1 - predicted)) + * @f] + * + * For a Softmax output layer it should be the categorical Cross-Entropy: + * @f[ + * result = -\sum target \cdot \log(predicted) + * @f] + * + * @param result A Scalar of the defined data type + */ + void (*crossentropy)(const aitensor_t *predicted_data, const aitensor_t *target_data, void *result); + + ///@} +}; + +/** @brief Cross-Entropy loss type + * + * Defines the type of the loss (for example for type checks and debug prints). + * See aicore_losstype for more information about the loss type. + */ +extern const aicore_losstype_t *ailoss_crossentropy_type; + +/** @brief Initialize and connect the given Cross-Entropy loss + * + * This function represents the "constructor" of the abstract Cross-Entropy loss. It initializes the loss structure + * and connects it to the output layer of the AIfES model.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example ailoss_crossentropy_f32_default()). + * + * @param *loss The loss to initialize. + * @param *input_layer The output layer of the model that provides the inputs to the loss. + * @return Pointer to the (successfully) initialized loss structure. + */ +ailoss_t *ailoss_crossentropy(ailoss_crossentropy_t *loss, ailayer_t *input_layer); + +/** @brief Calculate the combined derivative of the given Cross-Entropy loss and the output layer for error backpropagation + * + * *Implementation of ailoss.calc_delta.* + * + * It uses the result tensor of the output layer and the target data as input and writes the result + * to the deltas tensor (ailayer.deltas) of the output layer of the model. + * + * By combining the Cross-Entropy loss with a Sigmoid or Softmax output layer, the combined deltas can be + * calculated very efficiently. The backward function of the Sigmoid / Softmax layer is not used anymore. + * + * Calculation of the deltas: + * @f[ + * \delta_{in} \leftarrow p - y + * @f] + * + * \f$ \delta_{in} \f$: Result of the delta calculation of this loss (written to ailayer.deltas of the output layer of the model)\n + * \f$ p \f$: Result of the forward pass of the output layer of the model (predicted values)\n + * \f$ y \f$: Target data / True values / Labels\n\n + * + * Used math functions: + * * ailoss_crossentropy.tensor_sub + * + * @param *self Loss to calculate the deltas for + * @param *target_data Target data / True values / Labels + */ +void ailoss_crossentropy_calc_delta(ailoss_t *self, const aitensor_t *target_data); + +/** @brief Calculate the Cross-Entropy loss on the given target data + * + * *Implementation of ailoss.calc_loss.* + * + * It uses the result tensor of the output layer and the target data as input and writes the result + * to the given result scalar. + * + * Calculation of the loss with Sigmoid output layer: + * @f[ + * result \leftarrow - \sum_i (y_i \log(p_i) + (1 - y_i) \log(1 - p_i)) + * @f] + * + * Calculation of the loss with Softmax output layer: + * @f[ + * result \leftarrow - \sum_i y_i \log(p_i) + * @f] + * + * + * \f$ result \f$: Result of the loss calculation\n + * \f$ p \f$: Result of the forward pass of the output layer of the model (predicted values)\n + * \f$ y \f$: Target data / True values / Labels\n\n + * + * Used math functions: + * * ailoss_crossentropy.tensor_sub + * * ailoss_crossentropy.crossentropy + * + * @param *self Loss to calculate the deltas for + * @param *target_data Target data / True values / Labels + * @param *result Result scalar (the data type is specified by the data type specific implementations) + */ +void ailoss_crossentropy_calc_loss(ailoss_t *self, const aitensor_t *target_data, void *result); + +/** @brief Dummy backward-function for the output layer of the model + * + * *Implementation of ailayer.backward.* +* +* The ailoss_crossentropy_calc_delta() function calculates the combined delta for a Sigmoid or Softmax layer +* with Cross-Entropy loss. Therefor the backward function of the Sigmoid / Softmax layer is not needed anymore +* and gets exchanged by this dummy function. +* +* The dummy function does nothing. +* +* @param *self The output layer of the model +*/ +void ailoss_crossentropy_dummy_backward(ailayer_t *self); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the loss specification + * + * @param *self The loss to print the specification for + * @param *print Pointer to the print function to use + */ +void ailoss_crossentropy_print_specs(const ailoss_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // CROSSENTROPY_LOSS diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailoss/ailoss_mse.c b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailoss/ailoss_mse.c new file mode 100644 index 0000000..f42d37c --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailoss/ailoss_mse.c @@ -0,0 +1,99 @@ +/** + * \file basic/base/ailoss/ailoss_mse.c + * \version 2.0alpha + * \date 20.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/ailoss/ailoss_mse.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_losstype_t ailoss_mse_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "Mean Squared Error", + .print_specs = ailoss_mse_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_losstype_t *ailoss_mse_type = &ailoss_mse_type_s; + +ailoss_t *ailoss_mse(ailoss_mse_t *loss, ailayer_t *input_layer) +{ + loss->base.loss_type = ailoss_mse_type; + + loss->base.connection_layer.input_layer = input_layer; + input_layer->output_layer = &(loss->base.connection_layer); + + loss->base.loss_configuration = loss; + + loss->base.connection_layer.deltas.dtype = loss->dtype; + loss->base.connection_layer.deltas.dim = 2; + loss->base.connection_layer.deltas.shape = input_layer->result.shape; + loss->base.connection_layer.get_result_bound = 0; + + loss->base.calc_delta = ailoss_mse_calc_delta; + loss->base.calc_loss = ailoss_mse_calc_loss; + + return &loss->base; +} + +void ailoss_mse_calc_delta(ailoss_t *self, const aitensor_t *target_data) +{ + ailoss_mse_t *loss = (ailoss_mse_t *)(self->loss_configuration); + aitensor_t *predicted_data = &(self->connection_layer.input_layer->result); + + loss->tensor_sub(predicted_data, target_data, &(self->connection_layer.deltas)); + + // ToDo: Scale by batch size. (Divide / Shift) + + return; +} + +void ailoss_mse_calc_loss(ailoss_t *self, const aitensor_t *target_data, void *result) +{ + ailoss_mse_t *loss = (ailoss_mse_t *)(self->loss_configuration); + + aitensor_t *predicted_data = &(self->connection_layer.input_layer->result); + + uint8_t temp_tensor_data[aimath_sizeof_tensor_data(target_data)]; + aitensor_t temp_tensor = { + .dim = target_data->dim, + .shape = target_data->shape, + .data = temp_tensor_data, + .dtype = target_data->dtype, + .tensor_params = self->connection_layer.deltas.tensor_params + }; + + loss->tensor_sub(predicted_data, target_data, &temp_tensor); + loss->norm_squared(&temp_tensor, result); + + // ToDo: Scale by batch size. (Divide / Shift) + + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void ailoss_mse_print_specs(const ailoss_t *self, int (*print)(const char *format, ...)) +{ + return; +} +#endif diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailoss/ailoss_mse.h b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailoss/ailoss_mse.h new file mode 100644 index 0000000..bae272e --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/ailoss/ailoss_mse.h @@ -0,0 +1,162 @@ +/** + * \file basic/base/ailoss/ailoss_mse.h + * \internal + * \date 20.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link ailoss loss \endlink implementation of the Mean Squared Error (MSE) loss + * + * This is an "abstract" data-type independent implementation. To use the loss, use one of the provided + * implementations for a specific hardware and data-type (for example from ailoss_mse_default.h) or set + * the required math functions on your own. + * + * The Mean Squared Error (MSE) loss ist best suitable for regression tasks. The loss / cost is calculated as + * @f[ + * L(y, \hat{y}) = \sum_{i=0}^{N} (y_i - \hat{y}_i)^2 + * @f] + * with the predicted values \f$ \hat{y}_i \f$ and the target values \f$ y_i \f$. + * \f$ N \f$ is the number of elements of the \f$ y \f$ tensor. + * + * To get the "mean" normalization, you have to modify the learning rate to \f$ lr = \frac {1}{o \cdot n} \cdot lr \f$ + * with the number of outputs \f$ o \f$ and the batch size \f$ n \f$. + * + * The loss can be calculated with ailoss_mse_calc_loss(). For training the deltas /errors on the target data are + * calculated with ailoss_mse_calc_delta() and written to the deltas tensor of the connection layer. + */ + +#ifndef MSE_LOSS +#define MSE_LOSS + +#include "core/aifes_core.h" + +typedef struct ailoss_mse ailoss_mse_t; /**< New data type name for code reduction. */ + +/** @brief General \link ailoss_mse.h Mean Squared Error (MSE) loss \endlink struct + * + */ +struct ailoss_mse { + ailoss_t base; /**< Inherited field members from general ailoss struct. */ + const aimath_dtype_t *dtype; /**< Main data type of the loss. */ + + /** @name Math fuctions + * @brief Required data type specific math functions + */ + ///@{ + + /** @brief Required math function: Element wise tensor subtraction + * + * Requires a math function that subtracts two tensors element wise:\n + * @f[ + * result = a - b + * @f] + */ + void (*tensor_sub)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + /** @brief Required math function: Squared sum of tensor elements + * + * Requires a math function that calculates the squared sum of all elements of a given tensor:\n + * @f[ + * result = \sum_i x_{i}^2 + * @f] + */ + void (*norm_squared)(const aitensor_t *x, void *result); + + ///@} +}; + +/** @brief Mean Squared Error loss type + * + * Defines the type of the loss (for example for type checks and debug prints). + * See aicore_losstype for more information about the loss type. + */ +extern const aicore_losstype_t *ailoss_mse_type; + +/** @brief Initialize and connect the given MSE loss + * + * This function represents the "constructor" of the abstract MSE loss. It initializes the loss structure + * and connects it to the output layer of the AIfES model.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example ailoss_mse_f32_default()). + * + * @param *loss The loss to initialize. + * @param *input_layer The output layer of the model that provides the inputs to the loss. + * @return Pointer to the (successfully) initialized loss structure. + */ +ailoss_t *ailoss_mse(ailoss_mse_t *loss, ailayer_t *input_layer); + +/** @brief Calculate the derivative of the given MSE loss for error backpropagation + * + * *Implementation of ailoss.calc_delta.* + * + * It uses the result tensor of the output layer and the target data as input and writes the result + * to the deltas tensor (ailayer.deltas) of the connection layer (ailoss.connection_layer). + * + * Calculation of the deltas: + * @f[ + * \delta_{in} \leftarrow p - y + * @f] + * + * \f$ \delta_{in} \f$: Result of the delta calculation of this loss (written to ailayer.deltas of the ailoss.connection_layer)\n + * \f$ p \f$: Result of the forward pass of the output layer of the model (predicted values)\n + * \f$ y \f$: Target data / True values / Labels\n\n + * + * Used math functions: + * * ailoss_mse.tensor_sub + * + * @param *self Loss to calculate the deltas for + * @param *target_data Target data / True values / Labels + */ +void ailoss_mse_calc_delta(ailoss_t *self, const aitensor_t *target_data); + +/** @brief Calculate the MSE loss on the given target data + * + * *Implementation of ailoss.calc_loss.* + * + * It uses the result tensor of the output layer and the target data as input and writes the result + * to the given result scalar. + * + * Calculation of the loss: + * @f[ + * result \leftarrow \sum_i (y_i - p_i)^2 + * @f] + * + * \f$ result \f$: Result of the loss calculation\n + * \f$ p \f$: Result of the forward pass of the output layer of the model (predicted values)\n + * \f$ y \f$: Target data / True values / Labels\n\n + * + * Used math functions: + * * ailoss_mse.tensor_sub + * * ailoss_mse.norm_squared + * + * @param *self Loss to calculate the deltas for + * @param *target_data Target data / True values / Labels + * @param *result Result scalar (the data type is specified by the data type specific implementations) + */ +void ailoss_mse_calc_loss(ailoss_t *self, const aitensor_t *target_data, void *result); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the loss specification + * + * @param *self The loss to print the specification for + * @param *print Pointer to the print function to use + */ +void ailoss_mse_print_specs(const ailoss_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // MSE_LOSS diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aimath/aimath_basic.c b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aimath/aimath_basic.c new file mode 100644 index 0000000..e927504 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aimath/aimath_basic.c @@ -0,0 +1,78 @@ +/** + * \file basic/base/aimath/aimath_basic.c + * \version 2.0alpha + * \date 28.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Basic math operations for aitensor_t + * \details Basic functions for aitensor_t handling, e.g. calculation of size, printing etc. + */ + +#include "basic/base/aimath/aimath_basic.h" + +void print_aitensor(const aitensor_t *tensor) +{ + tensor->dtype->print_aitensor(tensor); + return; +} + +void print_aiscalar(const void *scalar, const aimath_dtype_t *dtype) +{ + dtype->print_aiscalar(scalar, printf); + return; +} + +void aimath_transpose_vector(aitensor_t *vector) +{ + uint16_t temp; + temp = vector->shape[0]; + vector->shape[0] = vector->shape[1]; + vector->shape[1] = temp; + return; +} + +uint32_t aimath_tensor_elements(const aitensor_t *tensor) +{ + uint32_t elems = 1; + uint8_t i; + + for(i = 0; i < tensor->dim; i++) + { + elems *= tensor->shape[i]; + } + return elems; +} + +uint32_t aimath_sizeof_dtype(const aimath_dtype_t *dtype) +{ + return dtype->size; +} + +uint32_t aimath_sizeof_tensor_data(const aitensor_t *tensor) +{ + return aimath_sizeof_dtype(tensor->dtype) * aimath_tensor_elements(tensor); +} + +uint32_t aimath_sizeof_tensor_params(const aitensor_t *tensor) +{ + return tensor->dtype->tensor_params_size; +} + +uint32_t aimath_sizeof_tensor(const aitensor_t *tensor) +{ + return sizeof(aitensor_t) + aimath_sizeof_tensor_data(tensor) + aimath_sizeof_tensor_params(tensor); +} diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aimath/aimath_basic.h b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aimath/aimath_basic.h new file mode 100644 index 0000000..178d702 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aimath/aimath_basic.h @@ -0,0 +1,123 @@ +/** + * \file basic/base/aimath/aimath_basic.h + * \internal + * \date 28.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Basic data-type independent math operations + * + */ + +#ifndef AIMATH_BASIC +#define AIMATH_BASIC + +#include "core/aifes_math.h" + +/** @brief Printing a tensor to console + * + * Calls the corresponding print function of the used aimath_dtype. + * + * Example: + * \code{.c} + * uint16_t tensor_shape[2] = {2, 3}; + * float tensor_data[2*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f}; + * aitensor_t tensor = AITENSOR_2D_F32(tensor_shape, tensor_data); + * + * print_aitensor(&tensor); + * \endcode + * + * @param *tensor The tensor to print. + */ +void print_aitensor(const aitensor_t *tensor); + +/** @brief Printing a scalar to console + * + * Calls the corresponding print function of the used aimath_dtype. + * + * Example: + * \code{.c} + * aiscalar_f32_t scalar = 42.0f; + * + * print_aiscalar(&scalar, aif32); + * \endcode + * + * @param *scalar The scalar to print. + * @param *dtype The data type of the scalar + */ +void print_aiscalar(const void *scalar, const aimath_dtype_t *dtype); + +/** @brief Transposes a vector + * + * @f[ + * vector \leftarrow vector^T + * @f] + * + * Make sure that the given tensor is a vector (with shape [1 x N] or [N x 1]) + * + * @param *vector The vector to transpose (2D tensor of shape [1 x N] or [N x 1]) + */ +void aimath_transpose_vector(aitensor_t *vector); + +/** @brief Calculates the number of elements in a tensor + * + * @param *tensor The tensor to count the elements of + * @return Number of elements in tensor + */ +uint32_t aimath_tensor_elements(const aitensor_t *tensor); + +/** @brief Returns the size of one value in the given data type in bytes + * + * @param *dtype The data type to get the size of + * @return Size of one value in the given data type in bytes + */ +uint32_t aimath_sizeof_dtype(const aimath_dtype_t *dtype); + +/** @brief Returns size of the data array of the tensor in bytes + * + * The size is calculated by: + * + * \code aimath_tensor_elements(tensor) * aimath_sizeof_dtype(tensor.dtype) \endcode + * + * @param *tensor The tensor to get the data size of + * @return Size of tensor data in bytes + */ +uint32_t aimath_sizeof_tensor_data(const aitensor_t *tensor); + +/** @brief Returns size of the aitensor.tensor_params for the data type of the tensor + * + * @param *tensor The tensor to get the parameters size of + * @return Size of tensor parameters in bytes + */ +uint32_t aimath_sizeof_tensor_params(const aitensor_t *tensor); + +/** @brief Returns size of tensor in bytes + * + * The size is calculated by: + * + * \code sizeof(aitensor_t) + aimath_sizeof_tensor_data(tensor) + aimath_sizeof_tensor_params(tensor) \endcode + * + * @param *tensor The tensor to get the size of + * @return Size of tensor in bytes + */ +uint32_t aimath_sizeof_tensor(const aitensor_t *tensor); + +//void aimath_create_tensor_2d(uint16_t shape_0, uint16_t shape_1, aimath_dtype_t dtype, void *memory_ptr); + +#endif // AIMATH_BASIC diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aimath/aimath_f32.c b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aimath/aimath_f32.c new file mode 100644 index 0000000..0eaa402 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aimath/aimath_f32.c @@ -0,0 +1,91 @@ +/** + * \file basic/base/aimath/aimath_f32.c + * \version 2.0alpha + * \date 28.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Basic functions for f32 datatypes + */ + +#include "basic/base/aimath/aimath_f32.h" + +const aimath_dtype_t aif32_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "F32", +#else + .name = 0, +#endif + .size = 4, + .tensor_params_size = 0, + .print_aitensor = aimath_f32_print_aitensor, + .print_aiscalar = aimath_f32_print_aiscalar +}; + +const aimath_dtype_t *aif32 = &aif32_s; + + +void aimath_f32_print_aitensor(const aitensor_t *tensor) +{ + uint16_t i, j, k, n; + printf("F32 [\n"); + if(tensor->dim == 1) + { + for(j = 0; j < tensor->shape[0]; j++) + { + printf("%10.5f\t", ((float *) tensor->data)[j]); + } + } + else if(tensor->dim == 2) + { + for(i = 0; i < tensor->shape[0]; i++) + { + for(j = 0; j < tensor->shape[1]; j++) + { + printf("%10.5f\t", ((float *) tensor->data)[i*tensor->shape[1] + j]); + } + printf("\n"); + } + } + else if(tensor->dim == 4) + { + float (*tensor_data)[tensor->shape[0]][tensor->shape[1]][tensor->shape[2]][tensor->shape[3]] = tensor->data; + + for(n = 0; n < tensor->shape[0]; n++) + { + for(k = 0; k < tensor->shape[1]; k++) + { + for(i = 0; i < tensor->shape[2]; i++) + { + for(j = 0; j < tensor->shape[3]; j++) + { + printf("%10.5f\t", (*tensor_data)[n][k][i][j]); + } + printf("\n"); + } + printf("\n"); + } + printf("\n"); + } + } + printf("]\n"); + return; +} + +void aimath_f32_print_aiscalar(const void *scalar, int (*print)(const char *format, ...)) +{ + print("%f (F32)", *((float *) scalar)); +} diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aimath/aimath_f32.h b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aimath/aimath_f32.h new file mode 100644 index 0000000..e067fe8 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aimath/aimath_f32.h @@ -0,0 +1,134 @@ +/** + * \file basic/base/aimath/aimath_f32.h + * \internal + * \date 28.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Definition of the F32 (aif32) data-type + * + * The F32 (aif32) data-type stores data as 32 bit single precision floating point values. + * It does not require any additional parameter to define the values. + * + * **Example: Create a F32 tensor**\n + * The tensor + * @f[ + * \left( \begin{array}{rrr} 0 & 1 & 2 \\ 3 & 4 & 5 \end{array}\right) + * @f] + * can be created with + * \code{.c} + * float example_data[] = {0.0f, 1.0f, 2.0f, + * 3.0f, 4.0f, 5.0f}; + * uint16_t example_shape[] = {2, 3}; + * aitensor_t example_tensor = { + * .dtype = aif32, + * .dim = 2, + * .shape = example_shape, + * .data = example_data + * }; + * \endcode + * + * **Example: Create a F32 scalar**\n + * Either create it as a normal float value + * \code{.c} + * float scalar = 42.0f; + * \endcode + * or with the alias + * \code{.c} + * aiscalar_f32_t scalar = 42.0f; + * \endcode + * + * **Example: Print a F32 tensor to the console** + * \code{.c} + * print_aitensor(&example_tensor); + * \endcode + * + * **Example: Print a F32 scalar to the console** + * \code{.c} + * print_aiscalar(&scalar, aif32); + * \endcode + */ + +#ifndef AIMATH_F32 +#define AIMATH_F32 + +#include "core/aifes_math.h" +#include "basic/base/aimath/aimath_basic.h" + +/** @brief Initialize a 2 dimensional F32 tensor + * + * @param shape A uint16_t array of length 2 for the shape + * @param data A float array for the tensor data + */ +#define AITENSOR_2D_F32(shape, data) {aif32, 2, shape, 0, data} + +/** @brief Scalar for F32 (aif32) data-type + * + * This is only an alias for a float value for consistency.\n + * You can create a F32 scalar either as a normal float value + * \code{.c} + * float scalar = 42.0f; + * \endcode + * or with the alias + * \code{.c} + * aiscalar_f32_t scalar = 42.0f; + * \endcode + * + * You can print the scalar to the console with + * \code{.c} + * print_aiscalar(&scalar, aif32); + * \endcode + * or by using + * \code{.c} + * aimath_f32_print_aiscalar(&scalar, printf); + * \endcode + */ +typedef float aiscalar_f32_t; + +/** @brief Printing a F32 tensor to console + * + * For users the function + * \code{.c} + * print_aitensor(&tensor); + * \endcode + * is prefered. + * + * @param *tensor The tensor to print. + */ +void aimath_f32_print_aitensor(const aitensor_t *tensor); + +/** @brief Printing a F32 scalar to console + * + * For users the function + * \code{.c} + * print_aiscalar(&scalar, aif32); + * \endcode + * is prefered. + * + * @param *scalar The scalar (type: float) to print. + * @param *print The print function to use + */ +void aimath_f32_print_aiscalar(const void *scalar, int (*print)(const char *format, ...)); + +/** @brief The F32 data-type indicator + * + * Use this variable to configure some element with the \link aimath_f32.h F32 \endlink data-type, + */ +extern const aimath_dtype_t *aif32; + +#endif // AIMATH_F32 diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aiopti/aiopti_adam.c b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aiopti/aiopti_adam.c new file mode 100644 index 0000000..74c7163 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aiopti/aiopti_adam.c @@ -0,0 +1,149 @@ +/** + * \file basic/base/aiopti/aiopti_adam.c + * \version 2.0alpha + * \date 12.11.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/aiopti/aiopti_adam.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_optitype_t aiopti_adam_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "ADAM", + .print_specs = aiopti_adam_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_optitype_t *aiopti_adam_type = &aiopti_adam_type_s; + +aiopti_t *aiopti_adam(aiopti_adam_t *opti) +{ + opti->base.optimizer_type = aiopti_adam_type; + opti->base.optimizer_configuration = opti; + + opti->base.sizeof_optimem = aiopti_adam_sizeof_optimem; + opti->base.init_optimem = aiopti_adam_init_optimem; + opti->base.zero_gradients = aiopti_adam_zero_gradients; + opti->base.update_params = aiopti_adam_update_params; + + return (&opti->base); +} + +uint32_t aiopti_adam_sizeof_optimem(aiopti_t *self, const aitensor_t *params){ + uint32_t memory = 0; + memory += sizeof(aiopti_adam_momentums_t); + memory += 2 * aimath_sizeof_tensor_data(params); + return memory; +} + +void aiopti_adam_init_optimem(aiopti_t *self, const aitensor_t *params, const aitensor_t *gradients, void *optimem) +{ + aiopti_adam_t *opti = (aiopti_adam_t *)(self->optimizer_configuration); + uint32_t address_counter = 0; + aiopti_adam_momentums_t *momentums = optimem; + address_counter += sizeof(aiopti_adam_momentums_t); + + momentums->m.dtype = gradients->dtype; + momentums->m.dim = gradients->dim; + momentums->m.shape = gradients->shape; + momentums->m.tensor_params = gradients->tensor_params; + momentums->m.data = optimem + address_counter; + address_counter += aimath_sizeof_tensor_data(&momentums->m); + + momentums->v.dtype = gradients->dtype; + momentums->v.dim = gradients->dim; + momentums->v.shape = gradients->shape; + momentums->v.tensor_params = gradients->tensor_params; + momentums->v.data = optimem + address_counter; + address_counter += aimath_sizeof_tensor_data(&momentums->v); + + opti->zero_tensor(&momentums->m); + opti->zero_tensor(&momentums->v); + + return; +} + +void aiopti_adam_zero_gradients(aiopti_t *self, aitensor_t *gradients) +{ + aiopti_adam_t *opti = (aiopti_adam_t *)(self->optimizer_configuration); + + opti->zero_tensor(gradients); + + return; +} + +void aiopti_adam_update_params(aiopti_t *self, aitensor_t *params, const aitensor_t *gradients, void *optimem) +{ + aiopti_adam_t *opti = (aiopti_adam_t *)(self->optimizer_configuration); + + aiopti_adam_momentums_t *momentums = optimem; + + uint8_t temp_tensor_data[aimath_sizeof_tensor_data(gradients)]; + aitensor_t temp_tensor = { + .dim = gradients->dim, + .shape = gradients->shape, + .data = temp_tensor_data, + .dtype = gradients->dtype, + .tensor_params = gradients->tensor_params + }; + + // m = beta1 * m + (1-beta1) * g + opti->scalar_mul(opti->beta1, &momentums->m, &momentums->m); + opti->scalar_mul(opti->one_minus_beta1, gradients, &temp_tensor); + opti->tensor_add(&momentums->m, &temp_tensor, &momentums->m); + + // v = beta2 * v + (1-beta2) * g*g + opti->scalar_mul(opti->beta2, &momentums->v, &momentums->v); + opti->multiply(gradients, gradients, &temp_tensor); + opti->scalar_mul(opti->one_minus_beta2, &temp_tensor, &temp_tensor); + opti->tensor_add(&momentums->v, &temp_tensor, &momentums->v); + + // lrt = lr * sqrt(1-beta2^t) / (1-beta1)^t + // params = params - lrt * m / (sqrt(v) + eps) + opti->sqrt(&momentums->v, &temp_tensor); + opti->scalar_add(opti->eps, &temp_tensor, &temp_tensor); + opti->divide(&momentums->m, &temp_tensor, &temp_tensor); + opti->scalar_mul(opti->lrt, &temp_tensor, &temp_tensor); + + temp_tensor.tensor_params = params->tensor_params; // switch params like shift to the parameter tensor params + opti->tensor_sub(params, &temp_tensor, params); + + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void aiopti_adam_print_specs(const aiopti_t *self, int (*print)(const char *format, ...)) +{ + aiopti_adam_t *self_casted = (aiopti_adam_t *) self->optimizer_configuration; + + print("lr: "); + self->dtype->print_aiscalar(self->learning_rate, print); + print("; beta1: "); + self->dtype->print_aiscalar(self_casted->beta1, print); + print("; beta2: "); + self->dtype->print_aiscalar(self_casted->beta2, print); + print("; eps: "); + self->dtype->print_aiscalar(self_casted->eps, print); + return; +} +#endif diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aiopti/aiopti_adam.h b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aiopti/aiopti_adam.h new file mode 100644 index 0000000..fb41ae3 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aiopti/aiopti_adam.h @@ -0,0 +1,295 @@ +/** + * \file basic/base/aiopti/aiopti_adam.h + * \internal + * \date 12.11.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link aiopti optimizer \endlink implementation of the [Adam optimizer](https://arxiv.org/pdf/1412.6980.pdf) + * + * This is an "abstract" data-type independent implementation. To use the optimizer, use one of the provided + * implementations for a specific hardware and data-type (for example from aiopti_adam_default.h) or set + * the required math functions on your own. + * + * The [Adam optimizer](https://arxiv.org/pdf/1412.6980.pdf) is based on SGD and uses first-order and second-order moments for adaptive estimation. + * It uses the pre-calculated gradients to optimize the given parameters. + * For every parameter \f$ p \f$ of the parameters to optimize (trainable parameters) and the related gradient \f$ g \f$ it calculates + * @f[ + * m_t = \beta_1 \cdot m_{t-1} + (1 - \beta_1) \cdot g_t + * @f] + * @f[ + * v_t = \beta_2 \cdot v_{t-1} + (1 - \beta_2) \cdot g^2_t + * @f] + * @f[ + * p_t = p_{t-1} - lr_t \cdot \frac{m_t} {\sqrt{v_t} + \hat{\epsilon}} + * @f] + * in every optimization step with \f$ lr_t = lr \cdot \frac{\sqrt{1 - \beta^t_2}} {(1 - \beta_1)^t} \f$.\n + * \f$ lr \f$ is the learning rate that defines how big the optimization steps should be, + * and therefore how fast the training will be. + * \f$ m \f$ and \f$ v \f$ are the first- and second-order moments related to the parameter and must be stored in the optimization memory + * for every parameter. + */ + +#ifndef AIOPTI_ADAM +#define AIOPTI_ADAM + +#include "core/aifes_core.h" + +typedef struct aiopti_adam aiopti_adam_t; /**< New data type name for code reduction. */ +typedef struct aiopti_adam_momentums aiopti_adam_momentums_t; /**< New data type name for code reduction. */ + +/** @brief General \link aiopti_adam.h Adam optimizer \endlink struct +* +*/ +struct aiopti_adam { + aiopti_t base; /**< Inherited field members from general optimizer struct. */ + + /** @name Optimizer configuration + * @brief Required configuration parameters for the optimizer + * + * These fields have to be configured by the user before calling the initializer function. + */ + ///@{ + void *beta1; /**< aiscalar: Exponential decay rate for the first moment estimates \f$ \beta_1 \f$ */ + void *beta2; /**< aiscalar: Exponential decay rate for the second moment estimates \f$ \beta_2 \f$ */ + void *eps; /**< aiscalar: Small positive number for numerical stability \f$ \hat{\epsilon} \f$ (avoid dividing by 0)*/ + ///@} + + /** @name Variables for internal computation + * + * These fields are automatically configured in the initializer function. + */ + ///@{ + void *beta1t; /**< aiscalar: Auxiliary variable to calculate \f$ \beta_1^t \f$ */ + void *beta2t; /**< aiscalar: Auxiliary variable to calculate \f$ \beta_2^t \f$ */ + void *one_minus_beta1; /**< aiscalar: Auxiliary variable to calculate \f$ (1 - \beta_1) \f$ */ + void *one_minus_beta2; /**< aiscalar: Auxiliary variable to calculate \f$ (1 - \beta_2) \f$ */ + void *lrt; /**< aiscalar: Auxiliary variable to calculate \f$ lr_t = lr \cdot \frac{\sqrt{1-\beta_2^t}}{(1-\beta_1^t)} \f$ */ + ///@} + + /** @name Math functions + * @brief Required data type specific math functions + */ + ///@{ + + /** @brief Required math function: Element wise tensor multiplication + * + * Requires a math function that multiplies two tensors element wise:\n + * @f[ + * result = a \odot b + * @f] + */ + void (*multiply)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + /** @brief Required math function: Element wise tensor dividation + * + * Requires a math function that devides two tensors element wise:\n + * @f[ + * result_i = a_i \div b_i + * @f] + */ + void (*divide)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + /** @brief Required math function: Element wise tensor addition + * + * Requires a math function that adds two tensors element wise:\n + * @f[ + * result = a + b + * @f] + */ + void (*tensor_add)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + /** @brief Required math function: Element wise tensor subtraction + * + * Requires a math function that subtracts two tensors element wise:\n + * @f[ + * result = a - b + * @f] + */ + void (*tensor_sub)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + /** @brief Required math function: Multiplication of a scalar with a tensor + * + * Requires a math function that multiplies a scalar with a tensor:\n + * @f[ + * result = scalar \cdot a + * @f] + */ + void (*scalar_mul)(const void *scalar, const aitensor_t *a, aitensor_t *result); + + /** @brief Required math function: Element wise addition of a scalar to a tensor + * + * Requires a math function that adds a scalar to every element of a tensor:\n + * @f[ + * result = scalar \cdot \left( \begin{array}{c} 1 & \cdots & 1 \\ \vdots & \ddots & \vdots \\ 1 & \cdots & 1 \\ \end{array}\right) + a + * @f] + */ + void (*scalar_add)(const void *scalar, const aitensor_t *a, aitensor_t *result); + + /** @brief Required math function: Square root + * + * Requires a math function that calculates the element wise square root of a tensor:\n + * @f[ + * result_{i} = \sqrt{x_i} + * @f] + */ + void (*sqrt)(const aitensor_t *x, aitensor_t *result); + + /** @brief Required math function: Sets the elements of a tensor to zero + * + * Requires a math function that sets the elements of the given tensor to zero (or to the zero_point):\n + * @f[ + * tensor_i = 0 + * @f] + */ + void (*zero_tensor)(aitensor_t *tensor); + + ///@{ +}; + +/** @brief Adam optimizer type + * + * Defines the type of the optimizer (for example for type checks and debug prints). + * See aicore_optitype for more information about the optimizer type. + */ +extern const aicore_optitype_t *aiopti_adam_type; + +/** @brief Struct for the momentum tensors of an Adam optimizer + * + * This structure is made for easy access of the momentum tensors on the optimization memory. + * + */ +struct aiopti_adam_momentums { + aitensor_t m; /**< First momentum vector. */ + aitensor_t v; /**< Second momentum vector. */ +}; + +/** @brief Initialize the given Adam optimizer + * + * This function represents the "constructor" of the abstract Adam optimizer.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example aiopti_adam_f32_default()). + * + * @param *opti The optimizer to initialize. + * @return Pointer to the (successfully) initialized general optimizer structure (aiopti_adam.base) + */ +aiopti_t *aiopti_adam(aiopti_adam_t *opti); + +/** @brief Calculates the required memory for the optimization step + * + * *Implementation of aiopti.sizeof_optimem.* + * + * Calculates the size of the memory space that must be reserved. + * The memory is used for the first and second moment tensors and is calculated by: + * + * \code 2 * (sizeof(aitensor) + sizeof(params.data)) \endcode + * + * @param *self The optimizer + * @param *params The tensor of trainable parameters to calculate the memory for + */ +uint32_t aiopti_adam_sizeof_optimem(aiopti_t *self, const aitensor_t *params); + +/** @brief Initialization of the optimization memory buffer + * + * *Implementation of aiopti.init_optimem.* + * + * Initialize the first and second moment tensors with zeros: + * @f[ + * m_{0,i} \leftarrow 0 + * @f] + * @f[ + * v_{0,i} \leftarrow 0 + * @f] + * + * Used math functions: + * * aiopti_adam.zero_tensor + * + * @param *self The optimizer + * @param *params The tensor of trainable parameters + * @param *gradients The gradients associated to the parameters + * @param *optimem The optimization memory (containing the first and second moment) associated to the parameters + */ +void aiopti_adam_init_optimem(aiopti_t *self, const aitensor_t *params, const aitensor_t *gradients, void *optimem); + +/** @brief Set the gradients to zero + * + * *Implementation of aiopti.zero_gradients.* + * + * @f[ + * g_{i} \leftarrow 0 + * @f] + * + * Used math functions: + * * aiopti_adam.zero_tensor + * + * @param *self The optimizer + * @param *gradients The gradients to set to zero + */ +void aiopti_adam_zero_gradients(aiopti_t *self, aitensor_t *gradients); + +/** @brief Update the given parameter tensor with respect to the gradients + * + * *Implementation of aiopti.update_params.* + * + * Calculate and update the values of the trainable parameters (perform one update step): + * @f[ + * m_t \leftarrow \beta_1 \cdot m_{t-1} + (1 - \beta_1) \cdot g_t + * @f] + * @f[ + * v_t \leftarrow \beta_2 \cdot v_{t-1} + (1 - \beta_2) \cdot g^2_t + * @f] + * @f[ + * p_t \leftarrow p_{t-1} - lr_t \cdot \frac{m_t} {\sqrt{v_t} + \hat{\epsilon}} + * @f] + * + * \f$ m \f$: First moment estimates\n + * \f$ v \f$: Second moment estimates\n + * \f$ p \f$: Tensor of trainable parameters to update (params)\n + * \f$ g \f$: Gradients\n + * \f$ lr \f$: Learning rate / Optimization step size\n + * \f$ \beta_1 \f$: Exponential decay rate for the first moment estimates\n + * \f$ \beta_2 \f$: Exponential decay rate for the second moment estimates\n + * \f$ \hat{\epsilon} \f$: Small positive number for numerical stability\n\n + * + * Used math functions: + * * aiopti_adam.scalar_mul + * * aiopti_adam.tensor_add + * * aiopti_adam.multiply + * * aiopti_adam.sqrt + * * aiopti_adam.scalar_add + * * aiopti_adam.divide + * * aiopti_adam.tensor_sub + * + * @param *self The optimizer + * @param *params The tensor of trainable parameters \f$ p \f$ to update + * @param *gradients The gradients \f$ g \f$ associated to the parameters + * @param *optimem The buffer to store the first and second momentums \f$ m \f$ and \f$ v \f$ + */ +void aiopti_adam_update_params(aiopti_t *self, aitensor_t *params, const aitensor_t *gradients, void *optimem); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the optimizer specification + * + * @param *self The optimizer to print the specification for + * @param *print Pointer to the print function to use + */ +void aiopti_adam_print_specs(const aiopti_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // AIOPTI_ADAM + diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aiopti/aiopti_sgd.c b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aiopti/aiopti_sgd.c new file mode 100644 index 0000000..42854fb --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aiopti/aiopti_sgd.c @@ -0,0 +1,158 @@ +/** + * \file basic/base/aiopti/aiopti_sgd.c + * \version 2.0alpha + * \date 20.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/base/aiopti/aiopti_sgd.h" +#include "basic/base/aimath/aimath_basic.h" + +const aicore_optitype_t aiopti_sgd_type_s = { +#ifdef AIDEBUG_PRINT_MODULE_SPECS + .name = "SGD", + .print_specs = aiopti_sgd_print_specs +#else + .name = 0, + .print_specs = 0 +#endif +}; +const aicore_optitype_t *aiopti_sgd_type = &aiopti_sgd_type_s; + +aiopti_t *aiopti_sgd(aiopti_sgd_t *opti) +{ + opti->base.optimizer_type = aiopti_sgd_type; + opti->base.optimizer_configuration = opti; + + opti->base.begin_step = 0; + opti->base.end_step = 0; + opti->base.zero_gradients = aiopti_sgd_zero_gradients; + + // Need to be set in the inherited implementation + opti->base.update_params = 0; + opti->base.sizeof_optimem = 0; + opti->base.init_optimem = 0; + + return &opti->base; +} + + +uint32_t aiopti_sgd_sizeof_optimem_with_momentum(aiopti_t *self, const aitensor_t *params){ + uint32_t memory = 0; + memory += sizeof(aitensor_t); + memory += aimath_sizeof_tensor_data(params); + return memory; +} + +uint32_t aiopti_sgd_sizeof_optimem_without_momentum(aiopti_t *self, const aitensor_t *params){ + return 0; +} + +void aiopti_sgd_init_optimem_with_momentum(aiopti_t *self, const aitensor_t *params, const aitensor_t *gradients, void *optimem) +{ + aiopti_sgd_t *opti = (aiopti_sgd_t *)(self->optimizer_configuration); + uint32_t address_counter = 0; + aitensor_t *v = (aitensor_t *) optimem; + address_counter += sizeof(aitensor_t); + + v->dtype = gradients->dtype; + v->dim = gradients->dim; + v->shape = gradients->shape; + v->tensor_params = gradients->tensor_params; + v->data = optimem + address_counter; + address_counter += aimath_sizeof_tensor_data(v); + + opti->zero_tensor(v); + + return; +} + +void aiopti_sgd_init_optimem_without_momentum(aiopti_t *self, const aitensor_t *params, const aitensor_t *gradients, void *optimem) +{ + return; +} + +void aiopti_sgd_zero_gradients(aiopti_t *self, aitensor_t *gradients) +{ + aiopti_sgd_t *opti = (aiopti_sgd_t *)(self->optimizer_configuration); + + opti->zero_tensor(gradients); + + return; +} + +void aiopti_sgd_update_params_with_momentum(aiopti_t *self, aitensor_t *params, const aitensor_t *gradients, void *optimem) +{ + aiopti_sgd_t *opti = (aiopti_sgd_t *)(self->optimizer_configuration); + + aitensor_t *v = (aitensor_t *) optimem; + + uint8_t temp_tensor_data[aimath_sizeof_tensor_data(params)]; + aitensor_t temp_tensor = { + .dim = gradients->dim, + .shape = gradients->shape, + .data = temp_tensor_data, + .dtype = gradients->dtype, + .tensor_params = gradients->tensor_params + }; + + // v = momentum * v + gradients + opti->scalar_mul(opti->momentum, v, &temp_tensor); + opti->tensor_add(&temp_tensor, gradients, v); + + // p = p - lr * v + opti->scalar_mul(opti->base.learning_rate, v, &temp_tensor); + opti->tensor_sub(params, &temp_tensor, params); + + return; +} + +void aiopti_sgd_update_params_without_momentum(aiopti_t *self, aitensor_t *params, const aitensor_t *gradients, void *optimem) +{ + aiopti_sgd_t *opti = (aiopti_sgd_t *)(self->optimizer_configuration); + + uint8_t temp_tensor_data[aimath_sizeof_tensor_data(params)]; + aitensor_t temp_tensor = { + .dim = gradients->dim, + .shape = gradients->shape, + .data = temp_tensor_data, + .dtype = gradients->dtype, + .tensor_params = gradients->tensor_params + }; + + // p = p - lr * g + opti->scalar_mul(opti->base.learning_rate, gradients, &temp_tensor); + opti->tensor_sub(params, &temp_tensor, params); + return; +} + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +void aiopti_sgd_print_specs(const aiopti_t *self, int (*print)(const char *format, ...)) +{ + aiopti_sgd_t *self_casted = (aiopti_sgd_t *) self->optimizer_configuration; + + print("lr: "); + self->dtype->print_aiscalar(self->learning_rate, print); + + print("; momentum: "); + self->dtype->print_aiscalar(self_casted->momentum, print); + return; +} +#endif diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aiopti/aiopti_sgd.h b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aiopti/aiopti_sgd.h new file mode 100644 index 0000000..f518278 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/base/aiopti/aiopti_sgd.h @@ -0,0 +1,272 @@ +/** + * \file basic/base/aiopti/aiopti_sgd.h + * \internal + * \date 20.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Base \link aiopti optimizer \endlink implementation of the Stochastic Gradient Descent (with momentum) optimizer + * + * This is an "abstract" data-type independent implementation. To use the optimizer, use one of the provided + * implementations for a specific hardware and data-type (for example from aiopti_sgd_default.h) or set + * the required math functions on your own. + * + * The Stochastic Gradient Descent (SGD) optimizer is the most basic optimizer in backpropagation. + * It uses the pre-calculated gradients to optimize the given parameters. In addition, a momentum term can be configured.\n + * For every parameter \f$ p \f$ of the parameters to optimize (trainable parameters) and the related gradient \f$ g \f$ it calculates + * @f[ + * p_t = p_{t-1} - lr \cdot g_t + * @f] + * if the momentum \f$ \mu = 0 \f$ and + * @f[ + * v_t = \mu \cdot v_{t-1} + g_t + * @f] + * @f[ + * p_t = p_{t-1} - lr \cdot v_t + * @f] + * if the momentum \f$ \mu \neq 0 \f$ in every optimization step.\n + * \f$ lr \f$ is the learning rate that defines how big the optimization steps should be, + * and therefore how fast the training will be. + * \f$ v \f$ is the momentum term or velocity related to the parameter and must be stored in the optimization memory + * for every parameter when momentum is set. + */ + +#ifndef AIOPTI_SGD +#define AIOPTI_SGD + +#include "core/aifes_core.h" + +typedef struct aiopti_sgd aiopti_sgd_t; /**< New data type name for code reduction. */ + +/** @brief General \link aiopti_sgd.h Stochastic Gradient Descent (SGD) optimizer \endlink struct +* +*/ +struct aiopti_sgd { + aiopti_t base; /**< Inherited field members from general optimizer struct. */ + + /** @name Optimizer configuration + * @brief Required configuration parameters for the optimizer + * + * These fields have to be configured by the user before calling the initializer function. + */ + ///@{ + void *momentum; /**< aiscalar: Momentum(set to null to save optimization memory) */ + ///@} + + /** @name Math functions + * @brief Required data type specific math functions + */ + ///@{ + + /** @brief Required math function: Element wise tensor subtraction + * + * Requires a math function that subtracts two tensors element wise:\n + * @f[ + * result = a - b + * @f] + */ + void (*tensor_sub)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + /** @brief Required math function: Element wise tensor addition + * + * Requires a math function that adds two tensors element wise:\n + * @f[ + * result = a + b + * @f] + */ + void (*tensor_add)(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + /** @brief Required math function: Multiplication of a scalar with a tensor + * + * Requires a math function that multiplies a scalar with a tensor:\n + * @f[ + * result = scalar \cdot a + * @f] + */ + void (*scalar_mul)(const void *scalar, const aitensor_t *a, aitensor_t *result); + + /** @brief Required math function: Sets the elements of a tensor to zero + * + * Requires a math function that sets the elements of the given tensor to zero (or to the zero_point):\n + * @f[ + * tensor_i = 0 + * @f] + */ + void (*zero_tensor)(aitensor_t *tensor); + + ///@} +}; + +/** @brief SGD optimizer type + * + * Defines the type of the optimizer (for example for type checks and debug prints). + * See aicore_optitype for more information about the optimizer type. + */ +extern const aicore_optitype_t *aiopti_sgd_type; + + +/** @brief Initialize the given SGD optimizer + * + * This function represents the "constructor" of the abstract SGD optimizer.\n + * This function is not intended to call it directly. Instead use one of the data type specific implementations + * (like for example aiopti_sgd_f32_default()). + * + * @param *opti The optimizer to initialize. + * @return Pointer to the (successfully) initialized general optimizer structure (aiopti_sgd.base) + */ +aiopti_t *aiopti_sgd(aiopti_sgd_t *opti); + +/** @brief Calculates the required memory for the optimization step when the momentum is not zero + * + * *Implementation of aiopti.sizeof_optimem.* + * + * Calculates the size of the memory space that must be reserved. + * The memory is used for the velocity tensor (momentum term) and is calculated by: + * + * \code sizeof(aitensor) + sizeof(params.data) \endcode + * + * @param *self The optimizer + * @param *params The tensor of trainable parameters to calculate the memory for + */ +uint32_t aiopti_sgd_sizeof_optimem_with_momentum(aiopti_t *self, const aitensor_t *params); + +/** @brief Calculates the required memory for the optimization step when the momentum is zero + * + * *Implementation of aiopti.sizeof_optimem.* + * + * Calculates the size of the memory space that must be reserved. + * The required memory is zero because no velocity term is needed. + * + * @param *self The optimizer + * @param *params The tensor of trainable parameters to calculae the memory for + */ +uint32_t aiopti_sgd_sizeof_optimem_without_momentum(aiopti_t *self, const aitensor_t *params); + +/** @brief Initialization of the optimization memory buffer when the momentum is not zero + * + * *Implementation of aiopti.init_optimem.* + * + * Initialize the velocity tensor with zeros: + * @f[ + * v_{0,i} \leftarrow 0 + * @f] + * + * Used math functions: + * * aiopti_sgd.zero_tensor + * + * @param *self The optimizer + * @param *params The tensor of trainable parameters + * @param *gradients The gradients associated to the parameters + * @param *optimem The optimization memory (containing the velocities) associated to the parameters + */ +void aiopti_sgd_init_optimem_with_momentum(aiopti_t *self, const aitensor_t *params, const aitensor_t *gradients, void *optimem); + +/** @brief Initialization of the optimization memory buffer when the momentum is zero + * + * *Implementation of aiopti.init_optimem.* + * + * Does nothing because no optimization memory is needed in this case. + * + * @param *self The optimizer + * @param *params The tensor of trainable parameters + * @param *gradients The gradients associated to the parameters + * @param *optimem The optimization memory associated to the parameters + */ +void aiopti_sgd_init_optimem_without_momentum(aiopti_t *self, const aitensor_t *params, const aitensor_t *gradients, void *optimem); + +/** @brief Set the gradients to zero + * + * *Implementation of aiopti.zero_gradients.* + * + * @f[ + * g_{i} \leftarrow 0 + * @f] + * + * Used math functions: + * * aiopti_sgd.zero_tensor + * + * @param *self The optimizer + * @param *gradients The gradients to set to zero + */ +void aiopti_sgd_zero_gradients(aiopti_t *self, aitensor_t *gradients); + +/** @brief Update the given parameter tensor with respect to the gradients when the momentum is not zero + * + * *Implementation of aiopti.update_params.* + * + * Calculate and update the values of the trainable parameters (perform one update step): + * @f[ + * v_t \leftarrow \mu \cdot v_{t-1} + g_t + * @f] + * @f[ + * p_t \leftarrow p_{t-1} - lr \cdot v_t + * @f] + * + * \f$ v \f$: Velocity tensor (momentum term), stored in the optimem\n + * \f$ p \f$: Tensor of trainable parameters to update (params)\n + * \f$ g \f$: Gradients\n + * \f$ lr \f$: Learning rate / Optimization step size\n + * \f$ \mu \f$: Momentum\n\n + * + * Used math functions: + * * aiopti_sgd.scalar_mul + * * aiopti_sgd.tensor_add + * * aiopti_sgd.tensor_sub + * + * @param *self The optimizer + * @param *params The tensor of trainable parameters \f$ p \f$ to update + * @param *gradients The gradients \f$ g \f$ associated to the parameters + * @param *optimem The buffer to store the velocity \f$ v \f$ + */ +void aiopti_sgd_update_params_with_momentum(aiopti_t *self, aitensor_t *params, const aitensor_t *gradients, void *optimem); + +/** @brief Update the given parameter tensor with respect to the gradients when the momentum is zero + * + * *Implementation of aiopti.update_params.* + * + * Calculate and update the values of the trainable parameters (perform one update step): + * @f[ + * p_t \leftarrow p_{t-1} - lr \cdot g_t + * @f] + * + * Used math functions: + * * aiopti_sgd.scalar_mul + * * aiopti_sgd.tensor_sub + * + * \f$ p \f$: Tensor of trainable parameters to update (params)\n + * \f$ g \f$: Gradients\n + * \f$ lr \f$: Learning rate / Optimization step size\n\n + * + * @param *self The optimizer + * @param *params The tensor of trainable parameters \f$ p \f$ to update + * @param *gradients The gradients \f$ g \f$ associated to the parameters + * @param *optimem Not required because no velocity is stored + */ +void aiopti_sgd_update_params_without_momentum(aiopti_t *self, aitensor_t *params, const aitensor_t *gradients, void *optimem); + +#ifdef AIDEBUG_PRINT_MODULE_SPECS +/** @brief Print the optimizer specification + * + * @param *self The optimizer to print the specification for + * @param *print Pointer to the print function to use + */ +void aiopti_sgd_print_specs(const aiopti_t *self, int (*print)(const char *format, ...)); +#endif // AIDEBUG_PRINT_MODULE_SPECS + +#endif // AIOPTI_SGD + diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/cmsis/ailayer/ailayer_dense_cmsis.c b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/cmsis/ailayer/ailayer_dense_cmsis.c new file mode 100644 index 0000000..a978db1 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/cmsis/ailayer/ailayer_dense_cmsis.c @@ -0,0 +1,49 @@ +/** + * \file basic/cmsis/ailayer/ailayer_dense_cmsis.c + * \version 2.0alpha + * \date 15.03.2021 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/ailayer/ailayer_dense_default.h" +#include "../../../aifes.h" + +#if __arm__ +#ifdef AIFES_WITH_CMSIS + +#include "basic/cmsis/ailayer/ailayer_dense_cmsis.h" +#include "basic/cmsis/aimath/aimath_f32_cmsis.h" + + +ailayer_t *ailayer_dense_f32_cmsis(ailayer_dense_t *layer, ailayer_t *input_layer) +{ + layer->result_dtype = aif32; + layer->weights_dtype = aif32; + layer->bias_dtype = aif32; + + layer->linear = aimath_f32_cmsis_linear; + layer->mat_mul = aimath_f32_cmsis_mat_mul; + layer->tensor_add = aimath_f32_default_tensor_add; + + return ailayer_dense(layer, input_layer); +} + +#endif // AIFES_WITH_CMSIS +#endif //__arm__ diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/cmsis/ailayer/ailayer_dense_cmsis.h b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/cmsis/ailayer/ailayer_dense_cmsis.h new file mode 100644 index 0000000..4945e23 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/cmsis/ailayer/ailayer_dense_cmsis.h @@ -0,0 +1,77 @@ +/** + * \file basic/cmsis/ailayer/ailayer_dense_cmsis.h + * \internal + * \date 15.03.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief [Arm CMSIS](https://developer.arm.com/tools-and-software/embedded/cmsis) implementation of the \link ailayer_dense.h Dense layer \endlink for Arm Cortex processors. + * + * Arm CMSIS implementations of the Dense layer in \link aimath_f32.h F32 \endlink data-type. + * These implementations are specifically designed for the Arm Cortex processors and take advantage of SIMD instructions. + * For more information about the Dense layer refer to ailayer_dense.h. + */ + +#ifndef AILAYER_DENSE_CMSIS +#define AILAYER_DENSE_CMSIS + +#include "../../../aifes.h" + +#if __arm__ +#ifdef AIFES_WITH_CMSIS + +#include "basic/base/ailayer/ailayer_dense.h" + +#include "basic/cmsis/aimath/aimath_f32_cmsis.h" + +/** @brief Initializes and connect a Dense layer with the \link aimath_f32.h F32 \endlink CMSIS implementation + * + * Example: Create the layer structure with pretrained weights:\n + * \code{.c} + * // Use constant data only for inference. For training remove the const qualifier!! + * const float weights_data_dense[] = {-10.1164f, -8.4212f, 5.4396f, 7.297f, -7.6482f, -9.0155f}; + * const float bias_data_dense[] = {-2.9653f, 2.3677f, -1.5968f}; + * ailayer_dense_f32_t dense_layer = { + * .neurons = 3, + * .weights.data = (float *) weights_data_dense, + * .bias.data = (float *) bias_data_dense + * }; + * \endcode + * + * Example: Create the layer structure for training:\n + * \code{.c} + * ailayer_dense_f32_t dense_layer = { + * .neurons = 3 + * }; + * \endcode + * + * Example: Initialize and connect the layer:\n + * \code{.c} + * x = ailayer_dense_f32_cmsis(&dense_layer, x); + * \endcode + * + * @param *layer The layer structure to initialize. + * @param *input_layer The prior layer. + * @return The (successfully) initialized layer structure. + */ +ailayer_t *ailayer_dense_f32_cmsis(ailayer_dense_t *layer, ailayer_t *input_layer); + +#endif // AILAYER_DENSE_CMSIS + +#endif // AIFES_WITH_CMSIS +#endif //__arm__ diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/cmsis/aimath/aimath_f32_cmsis.c b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/cmsis/aimath/aimath_f32_cmsis.c new file mode 100644 index 0000000..5f8a2bc --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/cmsis/aimath/aimath_f32_cmsis.c @@ -0,0 +1,104 @@ +/** + * \file basic/cmsis/aimath/aimath_f32_cmsis.c + * \version 2.0alpha + * \date 16.03.2021 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + + +#include "basic/cmsis/aimath/aimath_f32_cmsis.h" +#include "../../../aifes.h" + +#if __arm__ +#ifdef AIFES_WITH_CMSIS +#include "arm_math.h" + +/** +* Math CMSIS Matrix Multiplication and boradtcast Ass +* +* Matrixmultiplication using the CMSIS DSP Library and broadcast add using for loop +* +*/ +void aimath_f32_cmsis_linear(const aitensor_t *a, const aitensor_t *b, const aitensor_t *c, aitensor_t *result) +{ + +#ifdef SHAPE_CHECK + if(a->shape[1] != b->shape[0]) + { + LOG_E("MatMul input shapes doesn't match.\n"); + return; + } + if(a->shape[0] != result->shape[0] || b->shape[1] != result->shape[1]) + { + LOG_E("MatMul output shape doesn't match.\n"); + return; + } +#endif + + uint16_t i, j; + + float *c_data = c != 0 ? (float *) c->data : 0; + float *result_data = (float *) result->data; + + + aimath_f32_cmsis_mat_mul(a, b, result); + + + for(i = 0; i < result->shape[0]; i++) + { + for(j = 0; j < result->shape[1]; j++) + { + result_data[i*result->shape[1] + j] += c_data[j]; + } + } + + + + return; +} + + +/** +* Math CMSIS Matrix Multiplication +* +* Matrixmultiplication using the CMSIS DSP Library +* +* \ref https://arm-software.github.io/CMSIS_5/DSP/html/group__MatrixMult.html +*/ +void aimath_f32_cmsis_mat_mul(const aitensor_t *a, const aitensor_t *b, aitensor_t *result) +{ + float *a_data = (float *) a->data; + float *b_data = (float *) b->data; + float *result_data = (float *) result->data; + + + arm_matrix_instance_f32 a_mat; /* Matrix a Instance */ + arm_matrix_instance_f32 b_mat; /* Matrix b Instance */ + arm_matrix_instance_f32 result_mat; /* Matrix result Instance */ + + arm_mat_init_f32(&a_mat, a->shape[0], a->shape[1], a_data); + arm_mat_init_f32(&b_mat, b->shape[0], b->shape[1], b_data); + arm_mat_init_f32(&result_mat, result->shape[0], result->shape[1], result_data); + + + arm_mat_mult_f32(&a_mat, &b_mat, &result_mat); +} +#endif // AIFES_WITH_CMSIS +#endif // __arm__ \ No newline at end of file diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/cmsis/aimath/aimath_f32_cmsis.h b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/cmsis/aimath/aimath_f32_cmsis.h new file mode 100644 index 0000000..199bab1 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/cmsis/aimath/aimath_f32_cmsis.h @@ -0,0 +1,130 @@ +/** + * \file basic/cmsis/aimath/aimath_f32_cmsis.h + * \internal + * \date 15.03.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ +#ifndef AIMATH_F32_CMSIS +#define AIMATH_F32_CMSIS + + +#include "basic/base/aimath/aimath_f32.h" +#include "../../../aifes.h" + +#if __arm__ +#ifdef AIFES_WITH_CMSIS + + +/** @brief Performs a matrix multiplication of f32 tensors a and b and adds a 1D tensor c to each row, using the ARM CMSIS DSP + * + * @details The addition of the horizontal vector c is performed via broadcast, i.e. elementwise in each column + * Mathematically this broadcast is equal to multiplying c with an vertical vector of ones (with the same number of elements as c) + * + * @details + * and adding the result to a * b + * @f[ + * result = a \cdot b + \left( \begin{array}{r} + 1 \\ + 1 \\ + \vdots \\ + 1 \\ + \end{array}\right) \cdot c + * @f] + * + * Example: + * @f[ + * a = \left( \begin{array}{rrr} + 1 & 2 & 3 \\ + 4 & 5 & 6 \\ + 7 & 8 & 9 + \end{array}\right) + * @f] + * + * @f[ + * b = \left( \begin{array}{rr} + 1 & 0 \\ + 0 & 1 \\ + 0 & 0 + \end{array}\right) + * @f] + * + * @f[ + * c = \left( \begin{array}{rr} + 2 & 5 + \end{array}\right) + * @f] + * + * @f[ + * result = a \cdot b + \left( \begin{array}{r} + 1 \\ + 1 \\ + 1 \\ + \end{array}\right) \cdot c + * @f] + * + * @f[ + * = \left( \begin{array}{rr} + 1 & 2 \\ + 4 & 5 \\ + 7 & 8 + \end{array}\right) + + + \left( \begin{array}{rr} + 2 & 5 \\ + 2 & 5 \\ + 2 & 5 + \end{array}\right) + * @f] + * + * @details + * + * @f[ + * = \left( \begin{array}{rr} + 3 & 7 \\ + 6 & 10 \\ + 9 & 13 + \end{array}\right) + * @f] + * + * @param *a f32 tensor a + * @param *b f32 tensor b + * @param *c Tensor c, 1 row and as many columns as the result + * @param *result Result f32 tensor + */ +void aimath_f32_cmsis_linear(const aitensor_t *a, const aitensor_t *b, const aitensor_t *c, aitensor_t *result); + +/** @brief Performs a matrix multiplication of f32 tensors a and b, using the ARM CMSIS DSP + * + * @details + * @f[ + * result = a \cdot b + * @f] + * + * @param *a f32 tensor a + * @param *b f32 tensor b + * @param *result Result f32 tensor of the multiplication + */ +void aimath_f32_cmsis_mat_mul(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + +#endif // AIFES_WITH_CMSIS +#endif //__arm__ +#endif // AIMATH_F32_CMSIS diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_dense_default.c b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_dense_default.c new file mode 100644 index 0000000..96d6e85 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_dense_default.c @@ -0,0 +1,39 @@ +/** + * \file basic/default/ailayer/ailayer_dense_default.c + * \version 2.0alpha + * \date 28.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief See ailayer_dense_default.h for documentation. + * \details + */ + +#include "basic/default/ailayer/ailayer_dense_default.h" + + +ailayer_t *ailayer_dense_f32_default(ailayer_dense_f32_t *layer, ailayer_t *input_layer) +{ + layer->result_dtype = aif32; + layer->weights_dtype = aif32; + layer->bias_dtype = aif32; + + layer->linear = aimath_f32_default_linear; + layer->mat_mul = aimath_f32_default_mat_mul; + layer->tensor_add = aimath_f32_default_tensor_add; + + return ailayer_dense(layer, input_layer); +} diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_dense_default.h b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_dense_default.h new file mode 100644 index 0000000..26c31f2 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_dense_default.h @@ -0,0 +1,69 @@ +/** + * \file basic/default/ailayer/ailayer_dense_default.h + * \internal + * \date 28.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link ailayer_dense.h Dense layer \endlink + * + * Hardware independent implementations of the Dense layer in \link aimath_f32.h F32 \endlink data-type. + * For more information about the Dense layer refer to ailayer_dense.h. + */ + +#ifndef AILAYER_DENSE_DEFAULT +#define AILAYER_DENSE_DEFAULT + +#include "basic/base/ailayer/ailayer_dense.h" +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct ailayer_dense ailayer_dense_f32_t; + +/** @brief Initializes and connect a \link ailayer_dense.h Dense layer \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * Example: Create the layer structure with pretrained weights:\n + * \code{.c} + * // Use constant data only for inference. For training remove the const qualifier!! + * const float weights_data_dense[] = {-10.1164f, -8.4212f, 5.4396f, 7.297f, -7.6482f, -9.0155f}; + * const float bias_data_dense[] = {-2.9653f, 2.3677f, -1.5968f}; + * ailayer_dense_f32_t dense_layer = { + * .neurons = 3, + * .weights.data = (float *) weights_data_dense, + * .bias.data = (float *) bias_data_dense + * }; + * \endcode + * + * Example: Create the layer structure for training:\n + * \code{.c} + * ailayer_dense_f32_t dense_layer = { + * .neurons = 3 + * }; + * \endcode + * + * Example: Initialize and connect the layer:\n + * \code{.c} + * x = ailayer_dense_f32_default(&dense_layer, x); + * \endcode + * + * @param *layer The layer structure to initialize. + * @param *input_layer The prior layer. + * @return The (successfully) initialized layer structure. + */ +ailayer_t *ailayer_dense_f32_default(ailayer_dense_f32_t *layer, ailayer_t *input_layer); + +#endif // AILAYER_DENSE_DEFAULT diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_elu_default.c b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_elu_default.c new file mode 100644 index 0000000..d6b65ef --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_elu_default.c @@ -0,0 +1,40 @@ +/** + * \file basic/default/ailayer/ailayer_elu_default.c + * \version 2.0alpha + * \date 16.03.2021 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/ailayer/ailayer_elu_default.h" + +ailayer_t *ailayer_elu_f32_default(ailayer_elu_f32_t *layer, ailayer_t *input_layer) +{ + layer->base.dtype = aif32; + layer->base.alpha = &(layer->alpha); + + //forward + layer->base.elu = aimath_f32_default_elu; + + // backward + layer->base.d_elu = aimath_f32_default_d_elu; + layer->base.multiply = aimath_f32_default_multiply; + + return ailayer_elu(&layer->base, input_layer); +} diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_elu_default.h b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_elu_default.h new file mode 100644 index 0000000..e1548c4 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_elu_default.h @@ -0,0 +1,66 @@ +/** + * \file basic/default/ailayer/ailayer_elu_default.h + * \internal + * \date 16.03.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link ailayer_elu.h ELU layer \endlink + * + * Hardware independent implementations of the ELU layer in \link aimath_f32.h F32 \endlink data-type. + * For more information about the ELU layer refer to ailayer_elu.h. + */ +#ifndef AILAYER_ELU_DEFAULT +#define AILAYER_ELU_DEFAULT + +#include "basic/base/ailayer/ailayer_elu.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct ailayer_elu_f32 ailayer_elu_f32_t; + +/** @brief Data-type specific ELU layer struct for \link aimath_f32.h F32 \endlink + * + * Adds a data field for the constant alpha in \link aimath_f32.h F32 \endlink to the base implementation. + */ +struct ailayer_elu_f32 { + ailayer_elu_t base; /**< Inherited field members from general ailayer_elu struct. */ + aiscalar_f32_t alpha; /**< Data-type specific parameter used to calculate ELU function for input values < 0. */ +}; + +/** @brief Initializes and connect an \link ailayer_elu.h ELU layer \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * Example: Create the layer structure:\n + * \code{.c} + * ailayer_elu_f32_t elu_layer = { + * .alpha = 1.0f + * }; + * \endcode + * + * Example: Initialize and connect the layer:\n + * \code{.c} + * x = ailayer_elu_f32_default(&elu_layer, x); + * \endcode + * + * @param *layer The layer structure to initialize. + * @param *input_layer The prior layer. + * @return The (successfully) initialized layer structure. + */ +ailayer_t *ailayer_elu_f32_default(ailayer_elu_f32_t *layer, ailayer_t *input_layer); + +#endif // AILAYER_ELU_DEFAULT diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_input_default.c b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_input_default.c new file mode 100644 index 0000000..5d77f44 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_input_default.c @@ -0,0 +1,32 @@ +/** + * \file basic/default/ailayer/ailayer_input_default.c + * \version 2.0alpha + * \date 12.11.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/ailayer/ailayer_input_default.h" + +ailayer_t *ailayer_input_f32_default(ailayer_input_f32_t *layer) +{ + layer->dtype = aif32; + + return ailayer_input(layer); +} diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_input_default.h b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_input_default.h new file mode 100644 index 0000000..669fa47 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_input_default.h @@ -0,0 +1,59 @@ +/** + * \file basic/default/ailayer/ailayer_input_default.h + * \internal + * \date 19.11.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link ailayer_input.h Input layer \endlink + * + * Hardware independent implementations of the Input layer in \link aimath_f32.h F32 \endlink data-type. + * For more information about the Input layer refer to ailayer_input.h. + */ + +#ifndef AILAYER_INPUT_DEFAULT +#define AILAYER_INPUT_DEFAULT + +#include "basic/base/ailayer/ailayer_input.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct ailayer_input ailayer_input_f32_t; + +/** @brief Initializes and connect an \link ailayer_input.h Input layer \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * Example: Create the layer structure:\n + * \code{.c} + * uint16_t input_layer_shape[] = {1, 2}; + * ailayer_input_f32_t input_layer = { + * .input_dim = 2, + * .input_shape = input_layer_shape + * }; + * \endcode + * + * Example: Initialize and connect the layer:\n + * \code{.c} + * x = ailayer_input_f32_default(&input_layer); + * \endcode + * + * @param *layer The layer structure to initialize. + * @return The (successfully) initialized layer structure. + */ +ailayer_t *ailayer_input_f32_default(ailayer_input_f32_t *layer); + +#endif // AILAYER_INPUT_DEFAULT diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_leaky_relu_default.c b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_leaky_relu_default.c new file mode 100644 index 0000000..4219579 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_leaky_relu_default.c @@ -0,0 +1,40 @@ +/** + * \file basic/default/ailayer/ailayer_leaky_relu_default.c + * \version 2.0alpha + * \date 16.03.2021 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/ailayer/ailayer_leaky_relu_default.h" + +ailayer_t *ailayer_leaky_relu_f32_default(ailayer_leaky_relu_f32_t *layer, ailayer_t *input_layer) +{ + layer->base.dtype = aif32; + layer->base.alpha = &(layer->alpha); + + //forward + layer->base.leaky_relu = aimath_f32_default_leaky_relu; + + // backward + layer->base.d_leaky_relu = aimath_f32_default_d_leaky_relu; + layer->base.multiply = aimath_f32_default_multiply; + + return ailayer_leaky_relu(&layer->base, input_layer); +} diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_leaky_relu_default.h b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_leaky_relu_default.h new file mode 100644 index 0000000..e0199bd --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_leaky_relu_default.h @@ -0,0 +1,68 @@ + +/** + * \file basic/default/ailayer/ailayer_leaky_relu_default.h + * \internal + * \date 16.03.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link ailayer_leaky_relu.h Leaky ReLU layer \endlink + * + * Hardware independent implementations of the Leaky ReLU layer in \link aimath_f32.h F32 \endlink data-type. + * For more information about the Leaky ReLU layer refer to ailayer_leaky_relu.h. + */ + +#ifndef AILAYER_LEAKY_RELU_DEFAULT +#define AILAYER_LEAKY_RELU_DEFAULT + +#include "basic/base/ailayer/ailayer_leaky_relu.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct ailayer_leaky_relu_f32 ailayer_leaky_relu_f32_t; + +/** @brief Data-type specific Leaky ReLU layer struct for \link aimath_f32.h F32 \endlink + * + * Adds a data field for the constant alpha in \link aimath_f32.h F32 \endlink to the base implementation. + */ +struct ailayer_leaky_relu_f32 { + ailayer_leaky_relu_t base; /**< Inherited field members from general ailayer_leaky_relu struct. */ + aiscalar_f32_t alpha; /**< Data type-specific parameter used to calculate Leaky ReLU function for input values < 0. */ +}; + +/** @brief Initializes and connect a \link ailayer_leaky_relu.h Leaky ReLU layer \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * Example: Create the layer structure:\n + * \code{.c} + * ailayer_leaky_relu_f32_t leaky_relu_layer = { + * .alpha = 0.01f + * }; + * \endcode + * + * Example: Initialize and connect the layer:\n + * \code{.c} + * x = ailayer_leaky_relu_f32_default(&leaky_relu_layer, x); + * \endcode + * + * @param *layer the layer structure to be initialized + * @param *input_layer the prior layer that provides the input to this Leaky ReLU layer + * @return the initialized Leaky ReLU layer structure + */ +ailayer_t *ailayer_leaky_relu_f32_default(ailayer_leaky_relu_f32_t *layer, ailayer_t *input_layer); + +#endif // AILAYER_LEAKY_RELU_DEFAULT diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_relu_default.c b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_relu_default.c new file mode 100644 index 0000000..56b2b2d --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_relu_default.c @@ -0,0 +1,39 @@ +/** + * \file basic/default/ailayer/ailayer_relu_default.c + * \version 2.0alpha + * \date 07.12.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/ailayer/ailayer_relu_default.h" + +ailayer_t *ailayer_relu_f32_default(ailayer_relu_f32_t *layer, ailayer_t *input_layer) +{ + layer->dtype = aif32; + + //forward + layer->relu = aimath_f32_default_relu; + + // backward + layer->d_relu = aimath_f32_default_d_relu; + layer->multiply = aimath_f32_default_multiply; + + return ailayer_relu(layer, input_layer); +} diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_relu_default.h b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_relu_default.h new file mode 100644 index 0000000..83e2723 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_relu_default.h @@ -0,0 +1,56 @@ +/** + * \file basic/default/ailayer/ailayer_relu_default.h + * \internal + * \date 07.12.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link ailayer_relu.h ReLU layer \endlink + * + * Hardware independent implementations of the ReLU layer in \link aimath_f32.h F32 \endlink data-type. + * For more information about the ReLU layer refer to ailayer_relu.h. + */ + +#ifndef AILAYER_RELU_DEFAULT +#define AILAYER_RELU_DEFAULT + +#include "basic/base/ailayer/ailayer_relu.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct ailayer_relu ailayer_relu_f32_t; + +/** @brief Initializes and connect a \link ailayer_relu.h ReLU layer \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * Example: Create the layer structure:\n + * \code{.c} + * ailayer_relu_f32_t relu_layer; + * \endcode + * + * Example: Initialize and connect the layer:\n + * \code{.c} + * x = ailayer_relu_f32_default(&relu_layer, x); + * \endcode + * + * @param *layer The layer structure to initialize. + * @param *input_layer The prior layer. + * @return The (successfully) initialized layer structure. + */ +ailayer_t *ailayer_relu_f32_default(ailayer_relu_f32_t *layer, ailayer_t *input_layer); + +#endif // AILAYER_RELU_DEFAULT diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_sigmoid_default.c b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_sigmoid_default.c new file mode 100644 index 0000000..2ed7434 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_sigmoid_default.c @@ -0,0 +1,64 @@ +/** + * \file basic/default/ailayer/ailayer_sigmoid_default.c + * \version 2.0alpha + * \date 12.11.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/ailayer/ailayer_sigmoid_default.h" + +ailayer_t *ailayer_sigmoid_f32_default(ailayer_sigmoid_f32_t *layer, ailayer_t *input_layer) +{ + layer->dtype = aif32; + + //forward + layer->sigmoid = aimath_f32_default_sigmoid; + + // backward + layer->d_sigmoid = aimath_f32_default_d_sigmoid; + layer->multiply = aimath_f32_default_multiply; + + layer->base.get_result_bound = ailayer_sigmoid_get_result_bound_f32_default; + + return ailayer_sigmoid(layer, input_layer); +} + +uint8_t ailayer_sigmoid_get_result_bound_f32_default(const ailayer_t *self, const uint8_t selector, void *result_bound) +{ + float *bound = (float *) result_bound; + + switch(selector){ + case AILAYER_RESULT_LOWER_BOUND: + *bound = 0.0f; + return TRUE; + case AILAYER_RESULT_UPPER_BOUND: + *bound = 1.0f; + return TRUE; + case AILAYER_DELTAS_LOWER_BOUND: + return FALSE; + case AILAYER_DELTAS_UPPER_BOUND: + return FALSE; + default: + #ifdef AIDEBUG_PRINT_ERROR_MESSAGES + printf("\n+++ ERROR: Not defined result bound selector.\n"); + #endif // AIDEBUG_PRINT_ERROR_MESSAGES + return FALSE; + } +} diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_sigmoid_default.h b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_sigmoid_default.h new file mode 100644 index 0000000..1ee255f --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_sigmoid_default.h @@ -0,0 +1,73 @@ +/** + * \file basic/default/ailayer/ailayer_sigmoid_default.h + * \internal + * \date 12.11.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link ailayer_sigmoid.h Sigmoid layer \endlink + * + * Hardware independent implementations of the Sigmoid layer in \link aimath_f32.h F32 \endlink data-type. + * For more information about the Sigmoid layer refer to ailayer_sigmoid.h. + */ + +#ifndef AILAYER_SIGMOID_DEFAULT +#define AILAYER_SIGMOID_DEFAULT + +#include "basic/base/ailayer/ailayer_sigmoid.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct ailayer_sigmoid ailayer_sigmoid_f32_t; + +/** @brief Initializes and connect a \link ailayer_sigmoid.h Sigmoid layer \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * Example: Create the layer structure:\n + * \code{.c} + * ailayer_sigmoid_f32_t sigmoid_layer; + * \endcode + * + * Example: Initialize and connect the layer:\n + * \code{.c} + * x = ailayer_sigmoid_f32_default(&sigmoid_layer, x); + * \endcode + * + * @param *layer The layer structure to initialize. + * @param *input_layer The prior layer. + * @return The (successfully) initialized layer structure. + */ +ailayer_t *ailayer_sigmoid_f32_default(ailayer_sigmoid_f32_t *layer, ailayer_t *input_layer); + +/** @brief Get the value bounds for Sigmoid layers + * + * For the float 32 representation only result bounds are defined. + * + * @param *self Referenced layer structure. + * @param selector Select, which bound value to get. Possible values: + * - AILAYER_RESULT_LOWER_BOUND + * - AILAYER_RESULT_UPPER_BOUND + * - AILAYER_DELTAS_LOWER_BOUND + * - AILAYER_DELTAS_UPPER_BOUND + * @param result_bound Pointer to the memory space, the function writes the requested value to. + * @return + * - TRUE: requested value available and set to result_bound + * - FALSE: requestet value is _not_ available and result_bound is __not__ set + */ +uint8_t ailayer_sigmoid_get_result_bound_f32_default(const ailayer_t *self, const uint8_t selector, void *result_bound); + +#endif // AILAYER_SIGMOID_DEFAULT diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_softmax_default.c b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_softmax_default.c new file mode 100644 index 0000000..d393e3d --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_softmax_default.c @@ -0,0 +1,35 @@ +/** + * \file basic/default/ailayer/ailayer_softmax_default.c + * \version 2.0alpha + * \date 07.12.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/ailayer/ailayer_softmax_default.h" + +ailayer_t *ailayer_softmax_f32_default(ailayer_softmax_f32_t *layer, ailayer_t *input_layer) +{ + layer->dtype = aif32; + + //forward + layer->softmax = aimath_f32_default_softmax; + + return ailayer_softmax(layer, input_layer); +} diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_softmax_default.h b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_softmax_default.h new file mode 100644 index 0000000..cbb95c4 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_softmax_default.h @@ -0,0 +1,56 @@ +/** + * \file basic/default/ailayer/ailayer_softmax_default.h + * \internal + * \date 07.12.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link ailayer_softmax.h Softmax layer \endlink + * + * Hardware independent implementations of the Softmax layer in \link aimath_f32.h F32 \endlink data-type. + * For more information about the Softmax layer refer to ailayer_softmax.h. + */ + +#ifndef AILAYER_SOFTMAX_DEFAULT +#define AILAYER_SOFTMAX_DEFAULT + +#include "basic/base/ailayer/ailayer_softmax.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct ailayer_softmax ailayer_softmax_f32_t; + +/** @brief Initializes and connect an \link ailayer_softmax.h Softmax layer \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * Example: Create the layer structure:\n + * \code{.c} + * ailayer_softmax_f32_t softmax_layer; + * \endcode + * + * Example: Initialize and connect the layer:\n + * \code{.c} + * x = ailayer_softmax_f32_default(&softmax_layer, x); + * \endcode + * + * @param *layer The layer structure to initialize. + * @param *input_layer The prior layer. + * @return The (successfully) initialized layer structure. + */ +ailayer_t *ailayer_softmax_f32_default(ailayer_softmax_f32_t *layer, ailayer_t *input_layer); + +#endif // AILAYER_SOFTMAX_DEFAULT diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_softsign_default.c b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_softsign_default.c new file mode 100644 index 0000000..a766196 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_softsign_default.c @@ -0,0 +1,39 @@ +/** + * \file basic/default/ailayer/ailayer_softsign_default.c + * \version 2.0alpha + * \date 16.03.2021 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/ailayer/ailayer_softsign_default.h" + +ailayer_t *ailayer_softsign_f32_default(ailayer_softsign_f32_t *layer, ailayer_t *input_layer) +{ + layer->dtype = aif32; + + //forward + layer->softsign = aimath_f32_default_softsign; + + // backward + layer->d_softsign = aimath_f32_default_d_softsign; + layer->multiply = aimath_f32_default_multiply; + + return ailayer_softsign(layer, input_layer); +} diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_softsign_default.h b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_softsign_default.h new file mode 100644 index 0000000..06637d8 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_softsign_default.h @@ -0,0 +1,57 @@ +/** + * \file basic/default/ailayer/ailayer_softsign_default.h + * \internal + * \date 16.03.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link ailayer_softsign.h Softsign layer \endlink + * + * Hardware independent implementations of the Softsign layer in \link aimath_f32.h F32 \endlink data-type. + * For more information about the Softsign layer refer to ailayer_softsign.h. + */ + + +#ifndef AILAYER_SOFTSIGN_DEFAULT +#define AILAYER_SOFTSIGN_DEFAULT + +#include "basic/base/ailayer/ailayer_softsign.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct ailayer_softsign ailayer_softsign_f32_t; + +/** @brief Initializes and connect a \link ailayer_softsign.h Softsign layer \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * Example: Create the layer structure:\n + * \code{.c} + * ailayer_softsign_f32_t softsign_layer; + * \endcode + * + * Example: Initialize and connect the layer:\n + * \code{.c} + * x = ailayer_softsign_f32_default(&softsign_layer, x); + * \endcode + * + * @param *layer The layer structure to initialize. + * @param *input_layer The prior layer. + * @return The (successfully) initialized layer structure. + */ +ailayer_t *ailayer_softsign_f32_default(ailayer_softsign_f32_t *layer, ailayer_t *input_layer); + +#endif // AILAYER_SOFTSIGN_DEFAULT diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_tanh_default.c b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_tanh_default.c new file mode 100644 index 0000000..87c3fe6 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_tanh_default.c @@ -0,0 +1,67 @@ + +/** + * \file basic/default/ailayer/ailayer_tanh_default.c + * \version 2.0alpha + * \date 17.03.2021 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/ailayer/ailayer_tanh_default.h" + +ailayer_t *ailayer_tanh_f32_default(ailayer_tanh_f32_t *layer, ailayer_t *input_layer) +{ + layer->dtype = aif32; + + //forward + layer->tanh = aimath_f32_default_tanh; + + // backward + layer->d_tanh = aimath_f32_default_d_tanh; + layer->multiply = aimath_f32_default_multiply; + + layer->base.get_result_bound = ailayer_tanh_get_result_bound_f32_default; + + return ailayer_tanh(layer, input_layer); +} + +uint8_t ailayer_tanh_get_result_bound_f32_default(const ailayer_t *self, const uint8_t selector, void *result_bound) +{ + float *bound = (float *) result_bound; + + switch(selector){ + case AILAYER_RESULT_LOWER_BOUND: + *bound = -1.0f; + return TRUE; + case AILAYER_RESULT_UPPER_BOUND: + *bound = 1.0f; + return TRUE; + case AILAYER_DELTAS_LOWER_BOUND: + return FALSE; + case AILAYER_DELTAS_UPPER_BOUND: + return FALSE; + default: + #ifdef AIDEBUG_PRINT_ERROR_MESSAGES + printf("\n+++ ERROR: Not defined result bound selector.\n"); + #endif // AIDEBUG_PRINT_ERROR_MESSAGES + return FALSE; + } +} + + diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_tanh_default.h b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_tanh_default.h new file mode 100644 index 0000000..9e31ea8 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailayer/ailayer_tanh_default.h @@ -0,0 +1,70 @@ + +/** + * \file basic/default/ailayer/ailayer_tanh_default.h + * \internal + * \date 17.03.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link ailayer_tanh.h Tanh layer \endlink + * + * Hardware independent implementations of the Tanh layer in \link aimath_f32.h F32 \endlink data-type. + * For more information about the Tanh layer refer to ailayer_tanh.h. + */ + +#ifndef AILAYER_TANH_DEFAULT +#define AILAYER_TANH_DEFAULT + +#include "basic/base/ailayer/ailayer_tanh.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct ailayer_tanh ailayer_tanh_f32_t; + +/** @brief Initializes and connect an \link ailayer_tanh.h Tanh layer \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * Example: Create the layer structure:\n + * \code{.c} + * ailayer_tanh_f32_t tanh_layer; + * \endcode + * + * Example: Initialize and connect the layer:\n + * \code{.c} + * x = ailayer_tanh_f32_default(&tanh_layer, x); + * \endcode + * + * Initializes the tanh layer structure with the float 32 default implementation + * + * @param *layer The layer structure to initialize. + * @param *input_layer The prior layer. + * @return The (successfully) initialized layer structure. + */ +ailayer_t *ailayer_tanh_f32_default(ailayer_tanh_f32_t *layer, ailayer_t *input_layer); + +/** @brief Get valid value range of output of tanh layer +* +* Gives the upper and lower bound of the results obtained by interference with the tanh activation function +* +* @param *self the layer to get the upper and lower limit for +* @param *selector the type of the bound (result/deltas, upper/lower) +* @param *result_bound value of the limit, if exists +* @return TRUE or FALSE in case no limit exists +*/ +uint8_t ailayer_tanh_get_result_bound_f32_default(const ailayer_t *self, const uint8_t selector, void *result_bound); + +#endif // AILAYER_TANH_DEFAULT diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailoss/ailoss_crossentropy_default.c b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailoss/ailoss_crossentropy_default.c new file mode 100644 index 0000000..0a0dd99 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailoss/ailoss_crossentropy_default.c @@ -0,0 +1,48 @@ +/** + * \file basic/default/ailoss/ailoss_crossentropy_default.c + * \version 2.0alpha + * \date 14.01.2021 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/ailoss/ailoss_crossentropy_default.h" + + +ailoss_t *ailoss_crossentropy_f32_default(ailoss_crossentropy_f32_t *loss, ailayer_t *input_layer) +{ + loss->dtype = aif32; + + loss->tensor_sub = aimath_f32_default_tensor_sub; + + // Automatically set the right crossentropy function for the output layer. + if(input_layer->layer_type == ailayer_sigmoid_type){ + loss->crossentropy = aimath_f32_default_binary_crossentropy; + } else if (input_layer->layer_type == ailayer_softmax_type){ + loss->crossentropy = aimath_f32_default_categorical_crossentropy; + } else { + // Error: Input layer type not supported + #ifdef AIDEBUG_PRINT_ERROR_MESSAGES + printf("\n!!! Error: Input layer type not supported\n"); + #endif // AIDEBUG_PRINT_ERROR_MESSAGES + return 0; + } + + return ailoss_crossentropy(loss, input_layer); +} diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailoss/ailoss_crossentropy_default.h b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailoss/ailoss_crossentropy_default.h new file mode 100644 index 0000000..67e6421 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailoss/ailoss_crossentropy_default.h @@ -0,0 +1,73 @@ +/** + * \file basic/default/ailoss/ailoss_crossentropy_default.h + * \internal + * \date 14.01.2021 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link ailoss_crossentropy.h Cross-Entropy loss \endlink + * + * Hardware independent implementations of the Cross-Entropy loss in \link aimath_f32.h F32 \endlink data-type. + * For more information about the Cross-Entropy loss refer to ailoss_mse.h. + */ + +#ifndef AILOSS_CROSSENTROPY_DEFAULT +#define AILOSS_CROSSENTROPY_DEFAULT + +#include "basic/base/ailoss/ailoss_crossentropy.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct ailoss_crossentropy ailoss_crossentropy_f32_t; + +/** @brief Initializes and connect a \link ailoss_crossentropy.h Cross-Entropy loss \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * The labels must me either binary (when the output layer is a Sigmoid layer), for example + * @f[ + * \left( \begin{array}{ccc} 1 & 0 & 0 & 1 \\ 1 & 1 & 1 & 0 \\ 0 & 0 & 1 & 0 \end{array}\right) + * @f] + * + * or row wise one-hot encoded (when the output layer is a Softmax layer), for example + * @f[ + * \left( \begin{array}{ccc} 0 & 0 & 0 & 1 \\ 1 & 0 & 0 & 0 \\ 0 & 0 & 1 & 0 \end{array}\right) + * @f] + * + * If you want to provide labels as integers, please use ailoss_crossentropy_sparse8_f32_default() loss. + * + * Example: Create the loss structure:\n + * \code{.c} + * ailoss_crossentropy_f32_t crossentropy_loss; + * \endcode + * + * Example: Initialize and connect the loss to the layer structure:\n + * \code{.c} + * aimodel_t model; + * ... + * model.output_layer = ailayer_sigmoid_f32_default(&sigmoid_layer, x); + * + * model.loss = ailoss_crossentropy_f32_default(&crossentropy_loss, model.output_layer); + * \endcode + * + * @param *loss The loss structure to initialize. + * @param *input_layer The output layer of the model (**Must be either a Sigmoid or a Softmax layer!**). + * @return The (successfully) initialized loss structure. + */ +ailoss_t *ailoss_crossentropy_f32_default(ailoss_crossentropy_f32_t *loss, ailayer_t *input_layer); + +#endif // AILOSS_CROSSENTROPY_DEFAULT + diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailoss/ailoss_mse_default.c b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailoss/ailoss_mse_default.c new file mode 100644 index 0000000..7af3d78 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailoss/ailoss_mse_default.c @@ -0,0 +1,36 @@ +/** + * \file basic/default/ailoss/ailoss_mse_default.c + * \version 2.0alpha + * \date 28.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/ailoss/ailoss_mse_default.h" + + +ailoss_t *ailoss_mse_f32_default(ailoss_mse_f32_t *loss, ailayer_t *input_layer) +{ + loss->dtype = aif32; + + loss->tensor_sub = aimath_f32_default_tensor_sub; + loss->norm_squared = aimath_f32_default_norm_squared; + + return ailoss_mse(loss, input_layer); +} diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailoss/ailoss_mse_default.h b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailoss/ailoss_mse_default.h new file mode 100644 index 0000000..a294287 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/ailoss/ailoss_mse_default.h @@ -0,0 +1,58 @@ +/** + * \file basic/default/ailoss/ailoss_mse_default.h + * \internal + * \date 28.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link ailoss_mse.h Mean Squared Error loss \endlink + * + * Hardware independent implementations of the Mean Squared Error loss in \link aimath_f32.h F32 \endlink data-type. + * For more information about the Mean Squared Error loss refer to ailoss_mse.h. + */ + +#ifndef AILOSS_MSE_DEFAULT +#define AILOSS_MSE_DEFAULT + +#include "basic/base/ailoss/ailoss_mse.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct ailoss_mse ailoss_mse_f32_t; + +/** @brief Initializes and connect a \link ailoss_mse.h Mean Squared Error loss \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * Example: Create the loss structure:\n + * \code{.c} + * ailoss_mse_f32_t mse_loss; + * \endcode + * + * Example: Initialize and connect the loss to the layer structure:\n + * \code{.c} + * aimodel_t model; + * ... + * model.loss = ailoss_mse_f32_default(&mse_loss, model.output_layer); + * \endcode + * + * @param *loss The loss structure to initialize. + * @param *input_layer The output layer of the model. + * @return The (successfully) initialized loss structure. + */ +ailoss_t *ailoss_mse_f32_default(ailoss_mse_f32_t *loss, ailayer_t *input_layer); + +#endif // AILOSS_MSE_DEFAULT diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/aimath/aimath_f32_default.c b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/aimath/aimath_f32_default.c new file mode 100644 index 0000000..04c97d2 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/aimath/aimath_f32_default.c @@ -0,0 +1,551 @@ +/** + * \file basic/default/aimath/aimath_f32_default.c + * \version 2.0alpha + * \date 25.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/aimath/aimath_f32_default.h" +#include + + +void aimath_f32_default_linear(const aitensor_t *a, const aitensor_t *b, const aitensor_t *c, aitensor_t *result) +{ + uint16_t i, j, k; + float sum; + + float *a_data = (float *) a->data; + float *b_data = (float *) b->data; + float *c_data = c != 0 ? (float *) c->data : 0; + float *result_data = (float *) result->data; + +#ifdef SHAPE_CHECK + if(a->shape[1] != b->shape[0]) + { + LOG_E("MatMul input shapes doesn't match.\n"); + return; + } + if(a->shape[0] != result->shape[0] || b->shape[1] != result->shape[1]) + { + LOG_E("MatMul output shape doesn't match.\n"); + return; + } +#endif + + for(i = 0; i < a->shape[0]; i++) + { + for(j = 0; j < b->shape[1]; j++) + { + sum = 0.0f; + for(k = 0; k < a->shape[1]; k++) + { + sum += a_data[i*a->shape[1] + k] * b_data[k*b->shape[1] + j]; + } + if(c != 0){ + // Bias add + sum += c_data[j]; + } + result_data[i*b->shape[1] + j] = sum; + } + } + return; +} + +void aimath_f32_default_mat_mul(const aitensor_t *a, const aitensor_t *b, aitensor_t *result){ + aimath_f32_default_linear(a, b, 0, result); +} + +void aimath_f32_default_multiply(const aitensor_t *a, const aitensor_t *b, aitensor_t *result) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(a); i++) + { + ((float *) result->data)[i] = ((float *) a->data)[i] * ((float *) b->data)[i]; + } + return; +} + +void aimath_f32_default_divide(const aitensor_t *a, const aitensor_t *b, aitensor_t *result) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(a); i++) + { + ((float *) result->data)[i] = ((float *) a->data)[i] / ((float *) b->data)[i]; + } + return; +} + +void aimath_f32_default_scalar_mul(const void *scalar, const aitensor_t *a, aitensor_t *result) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(a); i++) + { + ((float *) result->data)[i] = *((float *) scalar) * ((float *) a->data)[i]; + } + return; +} + +void aimath_f32_default_scalar_add(const void *scalar, const aitensor_t *a, aitensor_t *result) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(a); i++) + { + ((float *) result->data)[i] = *((float *) scalar) + ((float *) a->data)[i]; + } + return; +} + +/** +* Add 2D tensors +*/ +void aimath_f32_default_tensor_add(const aitensor_t *a, const aitensor_t *b, aitensor_t *result) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(a); i++) + { + ((float *) result->data)[i] = ((float *) a->data)[i] + ((float *) b->data)[i]; + } + return; +} + +void aimath_f32_default_tensor_sub(const aitensor_t *a, const aitensor_t *b, aitensor_t *result) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(a); i++) + { + ((float *) result->data)[i] = ((float *) a->data)[i] - ((float *) b->data)[i]; + } + return; +} + +// only for 2D tensors +// a: f32 +// b: u8 +// result: f32 +void aimath_f32_default_tensor_sub_sparse8(const aitensor_t *a, const aitensor_t *b, aitensor_t *result) +{ + uint32_t i, index; + for(i = 0; i < a->shape[0]; i++) + { + index = a->shape[1] * i + ((uint8_t *) b->data)[i]; + ((float *) result->data)[index] = ((float *) a->data)[index] - 1.0f; + } + return; +} + +void aimath_f32_default_copy_tensor(const aitensor_t *from, aitensor_t *to) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(from); i++) + { + ((float *) to->data)[i] = ((float *) from->data)[i]; + } + return; +} + +void aimath_f32_default_transpose_vector(aitensor_t *vector) +{ + uint16_t temp; + temp = vector->shape[0]; + vector->shape[0] = vector->shape[1]; + vector->shape[1] = temp; + return; +} + +void aimath_f32_default_norm_squared(const aitensor_t *x, void *result) +{ + uint32_t i; + + *((float *) result) = 0.0f; + + for(i = 0; i < x->shape[0] * x->shape[1]; i++) + { + *((float *) result) += ((float *) x->data)[i] * ((float *) x->data)[i]; + } + return; +} + +void aimath_f32_default_sum(const aitensor_t *x, void *result) +{ + uint32_t i; + + *((float *) result) = 0.0f; + + for(i = 0; i < x->shape[0] * x->shape[1]; i++) + { + *((float *) result) += ((float *) x->data)[i]; + } + return; +} + +void aimath_f32_default_min(const aitensor_t *x, void *result) +{ + uint32_t i; + float min_value = FLT_MAX; + + for(i = 0; i < aimath_tensor_elements(x); i++) + { + if(((float *) x->data)[i] < min_value){ + min_value = ((float *) x->data)[i]; + } + } + *((float *) result) = min_value; + return; +} + + +void aimath_f32_default_max(const aitensor_t *x, void *result) +{ + uint32_t i; + float max_value = -FLT_MAX; + + for(i = 0; i < aimath_tensor_elements(x); i++) + { + if(((float *) x->data)[i] > max_value){ + max_value = ((float *) x->data)[i]; + } + } + *((float *) result) = max_value; + return; +} + + +void aimath_f32_default_sigmoid(const aitensor_t *x, aitensor_t *result) +{ + uint32_t i; + + for(i = 0; i < aimath_tensor_elements(x); i++) + { + ((float *) result->data)[i] = 1.0f / (1.0f + expf(- ((float *) x->data)[i])); + } + return; +} + +void aimath_f32_default_d_sigmoid(const aitensor_t *sigmoid_x, aitensor_t *result) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(sigmoid_x); i++) + { + // sigmoid'(x) = sigmoid(x) * (1 - sigmoid(x)) + ((float *) result->data)[i] = ((float *) sigmoid_x->data)[i] * (1.0f - ((float *) sigmoid_x->data)[i]); + } + return; +} + +void aimath_f32_default_tanh(const aitensor_t *x, aitensor_t *result) +{ + uint32_t i; + float temp; + for(i = 0; i < aimath_tensor_elements(x); i++) + { + temp = expf(((float *) x->data)[i]); + ((float *) result->data)[i] = (temp - (1.0f/temp)) / (temp + (1.0f/temp)); + } + return; +} + +void aimath_f32_default_d_tanh(const aitensor_t *tanh_x, aitensor_t *result) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(tanh_x); i++) + { + // tanh'(x) = 1 - (tanh(x))^2 + ((float *) result->data)[i] = 1.0f - (((float *) tanh_x->data)[i] * ((float *) tanh_x->data)[i]); + } + return; +} + +void aimath_f32_default_relu(const aitensor_t *x, aitensor_t *result) +{ + uint32_t i; + + for(i = 0; i < aimath_tensor_elements(x); i++) + { + ((float *) result->data)[i] = ((float *) x->data)[i] > 0.0f ? ((float *) x->data)[i] : 0.0f; + } + return; +} + +void aimath_f32_default_d_relu(const aitensor_t *x, aitensor_t *result) +{ + uint32_t i; + + for(i = 0; i < aimath_tensor_elements(x); i++) + { + ((float *) result->data)[i] = ((float *) x->data)[i] >= 0.0f ? 1.0f : 0.0f; + } + return; +} + +void aimath_f32_default_leaky_relu(const aitensor_t *x, const void *alpha, aitensor_t *result) +{ + uint32_t i; + + for(i = 0; i < aimath_tensor_elements(x); i++) + { + ((float *) result->data)[i] = ((float *) x->data)[i] >= 0.0f ? ((float *) x->data)[i] : ((float *) x->data)[i] * *((float *) alpha); + } + return; +} + +void aimath_f32_default_d_leaky_relu(const aitensor_t *x, const void *alpha, aitensor_t *result) +{ + uint32_t i; + + for(i = 0; i < aimath_tensor_elements(x); i++) + { + ((float *) result->data)[i] = ((float *) x->data)[i] >= 0.0f ? 1.0f : *((float *) alpha); + } + return; +} + +void aimath_f32_default_elu(const aitensor_t *x, const void *alpha, aitensor_t *result) +{ + uint32_t i; + + for(i = 0; i < aimath_tensor_elements(x); i++) + { + ((float *) result->data)[i] = ((float *) x->data)[i] > 0.0f ? ((float *) x->data)[i] : (*((float *) alpha) * (exp(((float *) x->data)[i]) - 1.0f)); + } + return; +} + +void aimath_f32_default_d_elu(const aitensor_t *x, const void *alpha, aitensor_t *result) +{ + uint32_t i; + + for(i = 0; i < aimath_tensor_elements(x); i++) + { + ((float *) result->data)[i] = ((float *) x->data)[i] > 0.0f ? 1.0f : (*((float *) alpha) * expf(((float *) x->data)[i])); + } + return; +} + +void aimath_f32_default_softmax(const aitensor_t *x, aitensor_t *result) +{ + uint32_t i, j; + float max; + float exp_sum; + + float *x_data = (float *) x->data; + float *result_data = (float *) result->data; + + // Multiplier for array index calculation + uint16_t multiplier = 1; + for(i = x->dim - 1; i >= 1; i--){ + multiplier *= x->shape[i]; + } + + // Do for every dataset. (0 is batch dimension) + for(i = 0; i < x->shape[0]; i++){ + // calc max value for numeric stability + max = x_data[0]; + for(j = 0; j < multiplier; j++) + { + if(x_data[i * multiplier + j] > max) max = x_data[i * multiplier + j]; + } + // calc exp functions + exp_sum = 0.0f; + for(j = 0; j < multiplier; j++) + { + result_data[i * multiplier + j] = aimath_f32_default_expf_fast(x_data[i * multiplier + j] - max); + exp_sum += result_data[i * multiplier + j]; + } + //calc softmax + for(j = 0; j < multiplier; j++) + { + result_data[i * multiplier + j] = result_data[i * multiplier + j] / exp_sum; + } + } + return; +} + +void aimath_f32_default_softsign(const aitensor_t *x, aitensor_t *result) +{ + uint32_t i; + + for(i = 0; i < aimath_tensor_elements(x); i++) + { + ((float *) result->data)[i] = ((float *) x->data)[i] / (1.0f + fabs(((float *) x->data)[i])); + } + return; +} + +void aimath_f32_default_d_softsign(const aitensor_t *x, aitensor_t *result) +{ + uint32_t i; + + for(i = 0; i < aimath_tensor_elements(x); i++) + { + ((float *) result->data)[i] = 1.0f / pow((1.0f + abs(((float *) x->data)[i])), 2); + } + return; +} + +// predicted data: f32 +// target_data: f32 +void aimath_f32_default_binary_crossentropy(const aitensor_t *predicted_data, const aitensor_t *target_data, void *result) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(predicted_data); i++) + { + if(((float *) target_data->data)[i] != 0){ + *((float *) result) -= ((float *) target_data->data)[i] * log(((float *) predicted_data->data)[i]) + + (1.0f - ((float *) target_data->data)[i]) * log(1.0f - ((float *) predicted_data->data)[i]); + } + } + return; +} + +// predicted data: f32 +// target_data: f32 +void aimath_f32_default_categorical_crossentropy(const aitensor_t *predicted_data, const aitensor_t *target_data, void *result) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(predicted_data); i++) + { + if(((float *) target_data->data)[i] != 0){ + *((float *) result) -= ((float *) target_data->data)[i] * log(((float *) predicted_data->data)[i]); + } + } + return; +} + +// 2D tensors only +// predicted data: f32 +// target_data: u8 +void aimath_f32_default_categorical_crossentropy_sparse8(const aitensor_t *predicted_data, const aitensor_t *target_data, void *result) +{ + uint32_t i, index; + for(i = 0; i < target_data->shape[0]; i++) + { + index = i * predicted_data->shape[1] + ((uint8_t *) target_data->data)[i]; + *((float *) result) -= log(((float *) predicted_data->data)[index]); + } + return; +} + +void aimath_f32_default_sqrt(const aitensor_t *x, aitensor_t *result) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(x); i++) + { + ((float *) result->data)[i] = sqrt(((float *) x->data)[i]); + } + return; +} + +void aimath_f32_default_zero_tensor(aitensor_t *tensor) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(tensor); i++) + { + ((float *) tensor->data)[i] = 0.0f; + } + return; +} + +void aimath_f32_default_init_zeros(aitensor_t *tensor) +{ + aimath_f32_default_zero_tensor(tensor); + return; +} + +void aimath_f32_default_tensor_init_uniform(aitensor_t *tensor, float from, float to) +{ + uint32_t i; + for(i = 0; i < aimath_tensor_elements(tensor); i++) + { + ((float *) tensor->data)[i] = ((float) rand() / (float) RAND_MAX) * (to - from) + from; + } + return; +} + +/* Glorot Uniform weight Initialization + * + * Glorot uniform initializer, also called Xavier uniform initializer. + * + * Glorot et al., 2010 + * + * \ref http://proceedings.mlr.press/v9/glorot10a/glorot10a.pdf + * \ref https://prateekvishnu.medium.com/xavier-and-he-normal-he-et-al-initialization-8e3d7a087528 + */ +void aimath_f32_default_init_glorot_uniform(aitensor_t *tensor) +{ + float fan_in, fan_out, fan_avg; + if(tensor->dim == 2) + { + fan_in = tensor->shape[0]; + fan_out = tensor->shape[1]; + } + else if(tensor->dim == 4) + { + fan_in = tensor->shape[1] * tensor->shape[2] * tensor->shape[3]; // In channel * kernel_elems + fan_out = tensor->shape[0] * tensor->shape[2] * tensor->shape[3]; // Out channel * kernel_elems + } + + fan_avg = (fan_in + fan_out) / 2.0f; + float r = sqrt(3.0f / fan_avg); + aimath_f32_default_tensor_init_uniform(tensor, -r, r); +} + +/* He Uniform weight Initialization + * + * He et al, 2015 + * + * \ref http://arxiv.org/abs/1502.01852 + * \ref https://prateekvishnu.medium.com/xavier-and-he-normal-he-et-al-initialization-8e3d7a087528 + */ +void aimath_f32_default_init_he_uniform(aitensor_t *tensor) +{ + float fan_in, fan_avg; + if(tensor->dim == 2) + { + fan_in = tensor->shape[0]; + } + else if(tensor->dim == 4) + { + fan_in = tensor->shape[1] * tensor->shape[2] * tensor->shape[3]; // In channel * kernel_elems + } + + fan_avg = fan_in / 2.0f; + float r = sqrt(3.0f / fan_avg); + aimath_f32_default_tensor_init_uniform(tensor, -r, r); +} + +//Info(?): http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.9.4508&rep=rep1&type=pdf +//ToDo: Mit dem in der Quelle erwhnten Parameter c rumspielen +float aimath_f32_default_expf_fast(float x) +{ + if(x > 80.0f){ + x = 80.0f; + } + else if(x < -80.0f){ + x = -80.0f; + } + + union { float f; int32_t x; } u; + //max-values for a ca. [-88 , 88] + // a = 2^23 / ln(2) ; b = 127 * 2^23 + // x_int32 = a * x_float + b + u.x = (int32_t) (12102203.0f * x + 1064631197.0f); + return u.f; + +} diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/aimath/aimath_f32_default.h b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/aimath/aimath_f32_default.h new file mode 100644 index 0000000..043209f --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/aimath/aimath_f32_default.h @@ -0,0 +1,1225 @@ +/** + * \file basic/default/aimath/aimath_f32_default.h + * \internal + * \date 25.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Math functions for \link aimath_f32.h F32 \endlink data type, default implementation + * + * These functions can be used when no hardware specific implementation is available. + */ + +#ifndef AIMATH_F32_DEFAULT +#define AIMATH_F32_DEFAULT + +#include +#include +#include +#include + +#include "basic/base/aimath/aimath_f32.h" + +/** @brief Performs a matrix multiplication of \link aimath_f32.h F32 \endlink matrices a and b and adds a vector c to each row + * + * The addition of the horizontal vector c is performed via broadcast, i.e. element wise in each column + * Mathematically this broadcast is equal to multiplying c with an vertical vector (with the same number of elements as c) + * and adding the result to a * b + * + * @f[ + * result = a \cdot b + \left( \begin{array}{c} + 1 \\ + 1 \\ + \vdots \\ + 1 \\ + \end{array}\right) \cdot c + * @f] + * + * Example: + * @f[ + * a = \left( \begin{array}{rrr} + 1 & 2 & 3 \\ + 4 & 5 & 6 \\ + 7 & 8 & 9 + \end{array}\right) + * @f] + * + * @f[ + * b = \left( \begin{array}{rr} + 1 & 0 \\ + 0 & 1 \\ + 0 & 0 + \end{array}\right) + * @f] + * + * @f[ + * c = \left( \begin{array}{rr} + 2 & 5 + \end{array}\right) + * @f] + * + * @f[ + * result = a \cdot b + \left( \begin{array}{r} 1 \\ + 1 \\ + 1 \\ + \end{array}\right) \cdot c + * @f] + * + * @f[ + * = \left( \begin{array}{rr} + 1 & 2 \\ + 4 & 5 \\ + 7 & 8 + \end{array}\right) + + \left( \begin{array}{rr} + 2 & 5 \\ + 2 & 5 \\ + 2 & 5 + \end{array}\right) + * @f] + * + * @details + * + * @f[ + * = \left( \begin{array}{rr} 3 & 7 \\ + 6 & 10 \\ + 9 & 13 + \end{array}\right) + * @f] + * + * Example: + * \code{.c} + * uint16_t a_shape[2] = {3, 3}; + * float a_data[3*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f, + * 7.0f, 8.0f, 9.0f}; + * aitensor_t a = AITENSOR_2D_F32(a_shape, a_data); + * + * uint16_t b_shape[2] = {3, 2}; + * float b_data[3*2] = {1.0f, 0.0f, + * 0.0f, 1.0f, + * 0.0f, 0.0f}; + * aitensor_t b = AITENSOR_2D_F32(b_shape, b_data); + * + * uint16_t c_shape[2] = {1, 2}; + * float c_data[1*2] = {2.0f, 5.0f}; + * aitensor_t c = AITENSOR_2D_F32(c_shape, c_data); + * + * uint16_t result_shape[2] = {3, 2}; + * float result_data[3*2]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_linear(&a, &b, &c, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *a Q31 matrix a (2D tensor of shape [N x K]) + * @param *b Q31 matrix b (2D tensor of shape [K x M]) + * @param *c Q31 vector c (2D tensor of shape [1 x M]) + * @param *result Resulting Q31 matrix (2D tensor of shape [N x M]) + */ +void aimath_f32_default_linear(const aitensor_t *a, const aitensor_t *b, const aitensor_t *c, aitensor_t *result); + +/** @brief Performs a matrix multiplication of \link aimath_f32.h F32 \endlink matrices a and b + * + * @f[ + * result = a \cdot b + * @f] + * + * Example: + * \code{.c} + * uint16_t a_shape[2] = {3, 3}; + * float a_data[3*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f, + * 7.0f, 8.0f, 9.0f}; + * aitensor_t a = AITENSOR_2D_F32(a_shape, a_data); + * + * uint16_t b_shape[2] = {3, 2}; + * float b_data[3*2] = {1.0f, 0.0f, + * 0.0f, 1.0f, + * 0.0f, 0.0f}; + * aitensor_t b = AITENSOR_2D_F32(b_shape, b_data); + * + * uint16_t result_shape[2] = {3, 2}; + * float result_data[3*2]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_mat_mul(&a, &b, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *a F32 matrix a (2D tensor of shape [N x K]) + * @param *b F32 matrix b (2D tensor of shape [K x M]) + * @param *result Resulting F32 matrix of the multiplication (2D tensor of shape [N x M]) + */ +void aimath_f32_default_mat_mul(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + +/** @brief Performs an element wise multiplication of \link aimath_f32.h F32 \endlink tensors a and b (Hadamard product) + * + * @f[ + * result = a \circ b + * @f] + * + * Example: + * \code{.c} + * uint16_t a_shape[2] = {2, 3}; + * float a_data[2*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f}; + * aitensor_t a = AITENSOR_2D_F32(a_shape, a_data); + * + * uint16_t b_shape[2] = {2, 3}; + * float b_data[2*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f}; + * aitensor_t b = AITENSOR_2D_F32(b_shape, b_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_multiply(&a, &b, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *a F32 tensor a (N-D tensor) + * @param *b F32 tensor b (N-D tensor) + * @param *result Resulting F32 tensor of the element wise multiplication (N-D tensor) + */ +void aimath_f32_default_multiply(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + +/** @brief Performs an element wise division of \link aimath_f32.h F32 \endlink tensors a and b (Hadamard division) + * + * @f[ + * result = a \oslash b + * @f] + * + * Example: + * \code{.c} + * uint16_t a_shape[2] = {2, 3}; + * float a_data[2*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f}; + * aitensor_t a = AITENSOR_2D_F32(a_shape, a_data); + * + * uint16_t b_shape[2] = {2, 3}; + * float b_data[2*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f}; + * aitensor_t b = AITENSOR_2D_F32(b_shape, b_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_divide(&a, &b, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *a F32 tensor a (N-D tensor) + * @param *b F32 tensor b (N-D tensor) + * @param *result Resulting F32 tensor of the element wise division (N-D tensor) + */ +void aimath_f32_default_divide(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + +/** @brief Performs a scalar multiplication (scaling) of \link aimath_f32.h F32 \endlink tensor a and a scalar + * + * @f[ + * result = scalar \cdot a + * @f] + * + * Example: + * \code{.c} + * uint16_t a_shape[2] = {2, 3}; + * float a_data[2*3] = {0.0f, 1.0f, 2.0f, + * 3.0f, 4.0f, 5.0f}; + * aitensor_t a = AITENSOR_2D_F32(a_shape, a_data); + * + * float scalar = 0.1f; + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_scalar_mul(&scalar, &a, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *scalar Scalar (type aiscalar_f32_t / float) + * @param *a F32 tensor a (N-D tensor) + * @param *result Resulting F32 tensor of the scalar multiplication (N-D tensor) + */ +void aimath_f32_default_scalar_mul(const void *scalar, const aitensor_t *a, aitensor_t *result); + +/** @brief Performs an element wise addition of a scalar to a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result = a + \left( \begin{array}{ccc} 1 & \ldots & 1 \\ + \vdots & \ddots & \vdots \\ + 1 & \ldots & 1 + \end{array}\right) \cdot scalar + * @f] + * + * Example: + * \code{.c} + * uint16_t a_shape[2] = {2, 3}; + * float a_data[2*3] = {0.0f, 1.0f, 2.0f, + * 3.0f, 4.0f, 5.0f}; + * aitensor_t a = AITENSOR_2D_F32(a_shape, a_data); + * + * float scalar = 0.1f; + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_scalar_add(&scalar, &a, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *scalar Scalar (type aiscalar_f32_t / float) + * @param *a F32 tensor a (N-D tensor) + * @param *result Resulting F32 tensor of the element wise scalar addition (N-D tensor) + */ +void aimath_f32_default_scalar_add(const void *scalar, const aitensor_t *a, aitensor_t *result); + +/** @brief Performs an element wise addition of \link aimath_f32.h F32 \endlink tensors a and b + * + * @f[ + * result = a + b + * @f] + * + * Example: + * \code{.c} + * uint16_t a_shape[2] = {2, 3}; + * float a_data[2*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f}; + * aitensor_t a = AITENSOR_2D_F32(a_shape, a_data); + * + * uint16_t b_shape[2] = {2, 3}; + * float b_data[2*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f}; + * aitensor_t b = AITENSOR_2D_F32(b_shape, b_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_tensor_add(&a, &b, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *a F32 tensor a (N-D tensor) + * @param *b F32 tensor b (N-D tensor) + * @param *result Resulting F32 tensor of the element wise addition (N-D tensor) + */ +void aimath_f32_default_tensor_add(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + + +/** @brief Performs a element wise subtraction of \link aimath_f32.h F32 \endlink tensors a and b + * + * @f[ + * result = a - b + * @f] + * + * Example: + * \code{.c} + * uint16_t a_shape[2] = {2, 3}; + * float a_data[2*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f}; + * aitensor_t a = AITENSOR_2D_F32(a_shape, a_data); + * + * uint16_t b_shape[2] = {2, 3}; + * float b_data[2*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f}; + * aitensor_t b = AITENSOR_2D_F32(b_shape, b_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_tensor_sub(&a, &b, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *a F32 tensor a (N-D tensor) + * @param *b F32 tensor b (N-D tensor) + * @param *result Resulting F32 tensor of the element wise subtraction (N-D tensor) + */ +void aimath_f32_default_tensor_sub(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + +/** @brief Performs a subtraction between a \link aimath_f32.h F32 \endlink matrix a and a \link aimath_u8.h U8 \endlink sparse matrix b + * + * This function can subtract a row wise one-hot encoded matrix in sparse representation + * (just the integer index of the 1 is stored) from a normal F32 matrix a. + * + * For example the matrix + * @f[ + * \left( \begin{array}{ccc} 0 & 0 & 0 & 1 \\ 1 & 0 & 0 & 0 \\ 0 & 0 & 1 & 0 \end{array}\right) + * @f] + * in sparse representation is + * @f[ + * \left( \begin{array}{ccc} 3 \\ 0 \\ 2 \end{array}\right) + * @f] + * + * The result is then calculated as + * @f[ + * result_{ij} = \begin{cases} + a_{ij} - 1 & \text{if } j = b_i\\ + a_{ij} & \text{if } j \neq b_i + \end{cases} + * @f] + * + * Example: + * \code{.c} + * uint16_t a_shape[2] = {2, 3}; + * float a_data[2*3] = {0.2f, 0.1f, 0.7f, + * 0.9f, 0.1f, 0.0f}; + * aitensor_t a = AITENSOR_2D_F32(a_shape, a_data); + * + * uint16_t b_shape[2] = {2, 1}; + * uint8_t b_data[2*1] = {2, + * 0}; + * aitensor_t b = AITENSOR_2D_U8(b_shape, b_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_tensor_sub_sparse8(&a, &b, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *a F32 matrix a (2D tensor of shape [N x M]) + * @param *b U8 sparse matrix b (2D tensor of shape [N x 1]) + * @param *result Resulting F32 tensor of the subtraction (2D tensor of shape [N x M]) + */ +void aimath_f32_default_tensor_sub_sparse8(const aitensor_t *a, const aitensor_t *b, aitensor_t *result); + +/** @brief Performs an element wise copy of \link aimath_f32.h F32 \endlink tensors + * + * @f[ + * to \leftarrow from + * @f] + * + * Dimension and shape of from and to tensors have to be the same. + * + * Example: + * \code{.c} + * uint16_t from_shape[2] = {2, 3}; + * float from_data[2*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f}; + * aitensor_t from = AITENSOR_2D_F32(from_shape, from_data); + * + * uint16_t to_shape[2] = {2, 3}; + * float to_data[2*3]; + * aitensor_t to = AITENSOR_2D_F32(to_shape, to_data); + * + * aimath_f32_default_copy_tensor(&from, &to); + * + * print_aitensor(&to); + * \endcode + * + * @param *from F32 tensor to copy from (N-D tensor) + * @param *to F32 tensor to copy to (N-D tensor) + */ +void aimath_f32_default_copy_tensor(const aitensor_t *from, aitensor_t *to); + +/** @brief Transposes a \link aimath_f32.h F32 \endlink vector + * + * The given tensor must be a vector (2D tensor of shape [1 x N] or [N x 1]). + * + * @f[ + * vector \leftarrow vector^T + * @f] + * + * Example: + * \code{.c} + * uint16_t vector_shape[2] = {1, 3}; + * float vector_data[1*3] = {1.0f, 2.0f, 3.0f}; + * aitensor_t vector = AITENSOR_2D_F32(vector_shape, vector_data); + * + * aimath_f32_default_transpose_vector(&vector); + * + * print_aitensor(&vector); + * \endcode + * + * @param *vector F32 vector (2D tensor of shape [1 x N] or [N x 1]) + */ +void aimath_f32_default_transpose_vector(aitensor_t *vector); + +/** @brief Calculates the squared sum of all elements in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result = \sum_i x_{i}^2 + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = {0.0f, 1.0f, 2.0f, + * 3.0f, 4.0f, 5.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * float result; + * + * aimath_f32_default_norm_squared(&x, &result); + * + * print_aiscalar(&result, aif32); + * \endcode + * + * @param *x F32 tensor x (N-D tensor) + * @param *result Scalar result (type aiscalar_f32_t / float) + */ +void aimath_f32_default_norm_squared(const aitensor_t *x, void *result); + +/** @brief Calculates the sum of all elements in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result = \sum_i x_{i} + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = {0.0f, 1.0f, 2.0f, + * 3.0f, 4.0f, 5.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * float result; + * + * aimath_f32_default_sum(&x, &result); + * + * print_aiscalar(&result, aif32); + * \endcode + * + * @param *x F32 tensor x (N-D tensor) + * @param *result Scalar result (type aiscalar_f32_t / float) + */ +void aimath_f32_default_sum(const aitensor_t *x, void *result); + +/** @brief Identifies the minimum value in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result = min(x) + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = {0.0f, 1.0f, 2.0f, + * 3.0f, 4.0f, 5.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * float result; + * + * aimath_f32_default_min(&x, &result); + * + * print_aiscalar(&result, aif32); + * \endcode + * + * @param *x F32 tensor x to get the minimum value of (N-D tensor) + * @param *result Scalar result (type aiscalar_f32_t / float) + */ +void aimath_f32_default_min(const aitensor_t *x, void *result); + +/** @brief Identifies the maximum value in a \link aimath_f32.h F32 \endlink tensor + * + * @details + * + * @f[ + * result = max(x) + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = {0.0f, 1.0f, 2.0f, + * 3.0f, 4.0f, 5.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * float result; + * + * aimath_f32_default_max(&x, &result); + * + * print_aiscalar(&result, aif32); + * \endcode + * + * @param *x F32 tensor x to get the maximum value of (N-D tensor) + * @param *result Scalar result (type aiscalar_f32_t / float) + */ +void aimath_f32_default_max(const aitensor_t *x, void *result); + +/** @brief Calculates the sigmoid of each element in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{i} = \sigma(x_{i}) = \frac{1}{1 + e^{-x_{i}}} + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_sigmoid(&x, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *x F32 tensor to calculate the sigmoid from (N-D tensor) + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_sigmoid(const aitensor_t *x, aitensor_t *result); + +/** @brief Calculates the derivative sigmoid of each element in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{i} = \sigma'(x_{i}) = \sigma(x_{i}) \cdot (1 - \sigma(x_{i})) + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_sigmoid(&x, &result); + * aimath_f32_default_d_sigmoid(&result, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *sigmoid_x F32 tensor with the sigmoid values \f$ \sigma(x_{i}) \f$ (N-D tensor) + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_d_sigmoid(const aitensor_t *sigmoid_x, aitensor_t *result); + +/** @brief Calculates the tanh of each element in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{i} = \tanh(x_{i}) = \frac{e^{x_i} - e^{-x_i}}{e^{x_i} + e^{-x_i}} + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_tanh(&x, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *x F32 tensor to calculate the tanh from (N-D tensor) + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_tanh(const aitensor_t *x, aitensor_t *result); + +/** @brief Calculates the tanh derivative of each element in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{i} = tanh'(x_{i}) = 1 - tanh(x_{i})^2 + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_tanh(&x, &result); + * aimath_f32_default_d_tanh(&result, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *tanh_x F32 tensor with the tanh values \f$ \tanh(x_{i}) \f$ (N-D tensor) + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_d_tanh(const aitensor_t *tanh_x, aitensor_t *result); + +/** @brief Calculates the rectifier (ReLU) value of each element in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{i} = max(0, x_{i}) + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_relu(&x, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *x F32 tensor to calculate the ReLU from (N-D tensor) + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_relu(const aitensor_t *x, aitensor_t *result); + + +/** @brief Calculates the rectifier (ReLU) derivative of each element in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{ij} = \begin{cases} + 0 & \text{if } x_i < 0\\ + 1 & \text{if } x_i \geq 0 + \end{cases} + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_d_relu(&x, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *x F32 tensor to calculate the ReLU derivative from (N-D tensor) + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_d_relu(const aitensor_t *x, aitensor_t *result); + +/** @brief Calculates the leaky rectifier (leaky ReLU) value of each element in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{i} = \begin{cases} + \alpha \cdot x_i & \text{if } x_i < 0 \\ + x_i & \text{if } x_i \geq 0 + \end{cases} + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * float alpha = 0.01f; + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_leaky_relu(&x, &alpha, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *x F32 tensor to calculate the leaky ReLU from (N-D tensor) + * @param *alpha Scalar \f$ \alpha \f$ (type aiscalar_f32_t / float) for the leakage + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_leaky_relu(const aitensor_t *x, const void *alpha, aitensor_t *result); + +/** @brief Calculates the leaky rectifier (leaky ReLU) derivative of each element in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{i} = \begin{cases} + \alpha & \text{if } x_i < 0\\ + 1 & \text{if } x_i \geq 0 + \end{cases} + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * float alpha = 0.01f; + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_d_leaky_relu(&x, &alpha, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *x F32 tensor to calculate the leaky ReLU derivative from (N-D tensor) + * @param *alpha Scalar \f$ \alpha \f$ (type aiscalar_f32_t / float) for the leakage + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_d_leaky_relu(const aitensor_t *x, const void *alpha, aitensor_t *result); + +/** @brief Calculates the exponential rectifier (ELU) value of each element in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{i} = \begin{cases} + \alpha \cdot (e^{x_i} - 1) & \text{if } x_i < 0 \\ + x_i & \text{if } x_i \geq 0 + \end{cases} + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * float alpha = 1.0f; + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_elu(&x, &alpha, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *x F32 tensor to calculate the ELU from (N-D tensor) + * @param *alpha Scalar \f$ \alpha \f$ (type aiscalar_f32_t / float) + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_elu(const aitensor_t *x, const void *alpha, aitensor_t *result); + +/** @brief Calculates the exponential rectifier (ELU) derivative of each element in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{i} = \begin{cases} + \alpha \cdot e^{x_i} & \text{if } x_i < 0\\ + 1 & \text{if } x_i \geq 0 + \end{cases} + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * float alpha = 1.0f; + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_d_elu(&x, &alpha, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *x F32 tensor to calculate the ELU derivative from (N-D tensor) + * @param *alpha Scalar \f$ \alpha \f$ (type aiscalar_f32_t / float) + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_d_elu(const aitensor_t *x, const void *alpha, aitensor_t *result); + +/** @brief Calculates the softmax value of each row of a \link aimath_f32.h F32 \endlink matrix + * + * @f[ + * result_{i} = \frac{e^{x_i}}{\sum_{j=1}^{K} e^{x_j}} + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_softmax(&x, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *x F32 matrix to calculate the softmax from (2D tensor) + * @param *result Resulting F32 matrix (2D tensor) + */ +void aimath_f32_default_softmax(const aitensor_t *x, aitensor_t *result); + +/** @brief Calculates the softsign value of each element in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{i} = \frac {x_i} {1 + |x_i|} + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_softsign(&x, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *x F32 tensor to calculate the softsign from (N-D tensor) + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_softsign(const aitensor_t *x, aitensor_t *result); + +/** @brief Calculates the softsign derivative of each element in a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{i} = \frac {x_i} {(1 + |x_i|)^2} + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = { 1.0f, -2.0f, 3.0f, + * -4.0f, 5.0f, -6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_d_softsign(&x, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *x F32 tensor to calculate the softsign derivative from (N-D tensor) + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_d_softsign(const aitensor_t *x, aitensor_t *result); + +/** @brief Calculates the binary cross entropy between the \link aimath_f32.h F32 \endlink predicted and the target data + * + * @f[ + * result = - \sum_i (target_i \cdot \log(predicted_i) + (1 - target_i) \cdot \log(1 - predicted_i)) + * @f] + * + * Example: + * \code{.c} + * uint16_t p_shape[2] = {2, 3}; + * float p_data[2*3] = {0.8f, 0.1f, 0.7f, + * 0.2f, 0.3f, 0.0f}; + * aitensor_t p = AITENSOR_2D_F32(p_shape, p_data); + * + * uint16_t t_shape[2] = {2, 3}; + * float t_data[2*3] = {1.0f, 0.0f, 1.0f, + * 0.0f, 0.0f, 0.0f}; + * aitensor_t t = AITENSOR_2D_F32(t_shape, t_data); + * + * float result; + * + * aimath_f32_default_binary_crossentropy(&p, &t, &result); + * + * print_aiscalar(&result, aif32); + * \endcode + * + * @param *predicted_data F32 matrix with the predicted or calculated values (2D tensor) + * @param *target_data F32 matrix with the target data / true values / labels (2D tensor with binary values 0 or 1) + * @param *result Resulting F32 matrix (2D tensor) + */ +void aimath_f32_default_binary_crossentropy(const aitensor_t *predicted_data, const aitensor_t *target_data, void *result); + +/** @brief Calculates the categorical cross entropy between the \link aimath_f32.h F32 \endlink predicted and the target data + * + * @f[ + * result = - \sum_i target_i \cdot \log(predicted_i) + * @f] + * + * Example: + * \code{.c} + * uint16_t p_shape[2] = {2, 3}; + * float p_data[2*3] = {0.2f, 0.1f, 0.7f, + * 0.9f, 0.1f, 0.0f}; + * aitensor_t p = AITENSOR_2D_F32(p_shape, p_data); + * + * uint16_t t_shape[2] = {2, 3}; + * float t_data[2*3] = {0.0f, 0.0f, 1.0f, + * 1.0f, 0.0f, 0.0f}; + * aitensor_t t = AITENSOR_2D_F32(t_shape, t_data); + * + * float result; + * + * aimath_f32_default_categorical_crossentropy(&p, &t, &result); + * + * print_aiscalar(&result, aif32); + * \endcode + * + * @param *predicted_data F32 matrix with the predicted or calculated values (2D tensor) + * @param *target_data F32 matrix with the target data / true values / labels (2D tensor, rows one-hot encoded) + * @param *result Resulting F32 matrix (2D tensor) + */ +void aimath_f32_default_categorical_crossentropy(const aitensor_t *predicted_data, const aitensor_t *target_data, void *result); + +/** @brief Calculates the categorical Cross-Entropy between the \link aimath_f32.h F32 \endlink predicted data and the \link aimath_u8.h U8 \endlink target data in sparse representation + * + * This function can calculate the crossentropy between a row wise one-hot encoded matrix in sparse representation + * (just the integer index of the 1 is stored) and a normal F32 matrix a. + * + * For example the matrix + * @f[ + * \left( \begin{array}{ccc} 0 & 0 & 0 & 1 \\ 1 & 0 & 0 & 0 \\ 0 & 0 & 1 & 0 \end{array}\right) + * @f] + * in sparse representation is + * @f[ + * \left( \begin{array}{ccc} 3 \\ 0 \\ 2 \end{array}\right) + * @f] + * + * The result is then calculated from the one-hot encoded target matrix: + * @f[ + * result = - \sum_i target_{one-hot,i} \cdot \log(predicted_i) + * @f] + * + * Example: + * \code{.c} + * uint16_t p_shape[2] = {2, 3}; + * float p_data[2*3] = {0.2f, 0.1f, 0.7f, + * 0.9f, 0.1f, 0.0f}; + * aitensor_t p = AITENSOR_2D_F32(p_shape, p_data); + * + * uint16_t t_shape[2] = {2, 1}; + * uint8_t t_data[2*1] = {2, + * 0}; + * aitensor_t t = AITENSOR_2D_U8(t_shape, t_data); + * + * float result; + * + * aimath_f32_default_categorical_crossentropy_sparse8(&p, &t, &result); + * + * print_aiscalar(&result, aif32); + * \endcode + * + * @param *predicted_data F32 matrix with the predicted or calculated values (2D tensor of shape [N x M]) + * @param *target_data U8 matrix with the target data / true values / labels (2D tensor of shape [N x 1] with true class indices) + * @param *result Resulting F32 matrix (2D tensor of shape [N x M]) + */ +void aimath_f32_default_categorical_crossentropy_sparse8(const aitensor_t *predicted_data, const aitensor_t *target_data, void *result); + +/** @brief Calculates the element wise square root of a \link aimath_f32.h F32 \endlink tensor + * + * @f[ + * result_{i} = \sqrt{x_{i}} + * @f] + * + * Example: + * \code{.c} + * uint16_t x_shape[2] = {2, 3}; + * float x_data[2*3] = {1.0f, 2.0f, 3.0f, + * 4.0f, 5.0f, 6.0f}; + * aitensor_t x = AITENSOR_2D_F32(x_shape, x_data); + * + * uint16_t result_shape[2] = {2, 3}; + * float result_data[2*3]; + * aitensor_t result = AITENSOR_2D_F32(result_shape, result_data); + * + * aimath_f32_default_sqrt(&x, &result); + * + * print_aitensor(&result); + * \endcode + * + * @param *x F32 tensor to calculate the square root from (N-D tensor) + * @param *result Resulting F32 tensor (N-D tensor) + */ +void aimath_f32_default_sqrt(const aitensor_t *x, aitensor_t *result); + +/** @brief Fills a \link aimath_f32.h F32 \endlink tensor with zeros + * + * @f[ + * tensor_{i} = 0 + * @f] + * + * Example: + * \code{.c} + * uint16_t tensor_shape[2] = {2, 3}; + * float tensor_data[2*3]; + * aitensor_t tensor = AITENSOR_2D_F32(tensor_shape, tensor_data); + * + * aimath_f32_default_zero_tensor(&tensor); + * + * print_aitensor(&tensor); + * \endcode + * + * In the F32 implementation of this function, there is no difference between aimath_f32_default_zero_tensor() + * and aimath_f32_default_init_zeros(). + * + * @param *tensor F32 tensor to set to zero (N-D tensor) + */ +void aimath_f32_default_zero_tensor(aitensor_t *tensor); + +/** @brief Fills a \link aimath_f32.h F32 \endlink tensor with zeros + * + * @f[ + * tensor_{i} = 0 + * @f] + * + * Example: + * \code{.c} + * uint16_t tensor_shape[2] = {2, 3}; + * float tensor_data[2*3]; + * aitensor_t tensor = AITENSOR_2D_F32(tensor_shape, tensor_data); + * + * aimath_f32_default_init_zeros(&tensor); + * + * print_aitensor(&tensor); + * \endcode + * + * In the F32 implementation of this function, there is no difference between aimath_f32_default_zero_tensor() + * and aimath_f32_default_init_zeros(). + * + * @param *tensor F32 tensor to set to zero (N-D tensor) + */ +void aimath_f32_default_init_zeros(aitensor_t *tensor); + +/** @brief Fills a \link aimath_f32.h F32 \endlink tensor with random numbers created from a uniform distribution within given range + * + * @f[ + * tensor_i \in \mathcal{U(from, to)} + * @f] + * + * Example: + * \code{.c} + * uint16_t tensor_shape[2] = {2, 3}; + * float tensor_data[2*3]; + * aitensor_t tensor = AITENSOR_2D_F32(tensor_shape, tensor_data); + * + * aimath_f32_default_tensor_init_uniform(&tensor, -1.5f, 1.5f); + * + * print_aitensor(&tensor); + * \endcode + * + * @param *tensor F32 tensor to initialize with random numbers (N-D tensor) + * @param from Minimum value of the uniform distribution + * @param to Maximum value of the uniform distribution + */ +void aimath_f32_default_tensor_init_uniform(aitensor_t *tensor, float from, float to); + +/** @brief Fills a \link aimath_f32.h F32 \endlink tensor with random numbers uniformly within given range, according to Glorot et al. + * + * @f[ + * fan_{avg} = \frac{fan_{in} + fan_{out}}{2} + * @f] + * @f[ + * r = \sqrt{\frac{3}{fan_{avg}}} + * @f] + * @f[ + * tensor_i \in \mathcal{U(-r, r)} + * @f] + * + * Example: + * \code{.c} + * uint16_t tensor_shape[2] = {2, 3}; + * float tensor_data[2*3]; + * aitensor_t tensor = AITENSOR_2D_F32(tensor_shape, tensor_data); + * + * aimath_f32_default_init_glorot_uniform(&tensor); + * + * print_aitensor(&tensor); + * \endcode + * + * @see Glorot et al., 2010 ( http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf ) + * + * @param *tensor F32 tensor to initialize with random numbers (N-D tensor) + */ +void aimath_f32_default_init_glorot_uniform(aitensor_t *tensor); + +/** @brief Fills a \link aimath_f32.h F32 \endlink tensor with uniformly drawn random numbers within given range, according to He et al. + * + * @f[ + * fan_{avg} = \frac{fan_{in}}{2} + * @f] + * @f[ + * r = \sqrt{\frac{3}{fan_{avg}}} + * @f] + * @f[ + * tensor_i \in \mathcal{U(-r, r)} + * @f] + * + * Example: + * \code{.c} + * uint16_t tensor_shape[2] = {2, 3}; + * float tensor_data[2*3]; + * aitensor_t tensor = AITENSOR_2D_F32(tensor_shape, tensor_data); + * + * aimath_f32_default_init_he_uniform(&tensor); + * + * print_aitensor(&tensor); + * \endcode + * + * @see He et al., 2015 ( https://www.cv-foundation.org/openaccess/content_iccv_2015/html/He_Delving_Deep_into_ICCV_2015_paper.html ) + * + * @param *tensor F32 tensor to initialize with random numbers (N-D tensor) + */ +void aimath_f32_default_init_he_uniform(aitensor_t *tensor); + +/** @brief Fast approximation of the exponential function + * + * @see http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.9.4508&rep=rep1&type=pdf + * + * @param x Input of the exponential function + */ +float aimath_f32_default_expf_fast(const float x); + +#endif // AIMATH_F32_DEFAULT + diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/aiopti/aiopti_adam_default.c b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/aiopti/aiopti_adam_default.c new file mode 100644 index 0000000..0f42a68 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/aiopti/aiopti_adam_default.c @@ -0,0 +1,88 @@ +/** + * \file basic/default/aiopti/aiopti_adam_default.c + * \version 2.0alpha + * \date 12.11.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/aiopti/aiopti_adam_default.h" + +aiopti_t *aiopti_adam_f32_default(aiopti_adam_f32_t *opti) +{ + opti->base.base.dtype = aif32; + + opti->base.base.learning_rate = &opti->learning_rate; + + opti->base.beta1 = &opti->beta1; + opti->base.beta2 = &opti->beta2; + opti->base.eps = &opti->eps; + + opti->base.beta1t = &opti->beta1t; + opti->base.beta2t = &opti->beta2t; + opti->base.one_minus_beta1 = &opti->one_minus_beta1; + opti->base.one_minus_beta2 = &opti->one_minus_beta2; + opti->base.lrt = &opti->lrt; + + // ToDo: Check Initialization + opti->beta1t = opti->beta1; + opti->beta2t = opti->beta2; + opti->one_minus_beta1 = 1.0f - opti->beta1; + opti->one_minus_beta2 = 1.0f - opti->beta2; + //opti->lrt = *((float *) opti->base.base.learning_rate) * sqrtf(1.0f - opti->beta2) / (1.0f - opti->beta1); + + // Set f32 function of optimizer base function begin_step() and end_step() + opti->base.base.begin_step = aiopti_adam_f32_default_begin_step; + opti->base.base.end_step = aiopti_adam_f32_default_end_step; + + // Set f32 math functions of adam optimizer + opti->base.multiply = aimath_f32_default_multiply; + opti->base.divide = aimath_f32_default_divide; + opti->base.tensor_add = aimath_f32_default_tensor_add; + opti->base.tensor_sub = aimath_f32_default_tensor_sub; + opti->base.scalar_mul = aimath_f32_default_scalar_mul; + opti->base.scalar_add = aimath_f32_default_scalar_add; + opti->base.sqrt = aimath_f32_default_sqrt; + + opti->base.zero_tensor = aimath_f32_default_zero_tensor; + + return aiopti_adam(&opti->base); +} + +void aiopti_adam_f32_default_begin_step(aiopti_t *self) +{ + aiopti_adam_t *opti = (aiopti_adam_t *)(self->optimizer_configuration); + + // lr_t = lr * sqrt(1-beta2^t) / (1-beta1^t) + *((float *) opti->lrt) = *((float *) self->learning_rate) * sqrtf(1.0f - *((float *) opti->beta2t)) / (1.0f - *((float *) opti->beta1t)); + return; +} + +void aiopti_adam_f32_default_end_step(aiopti_t *self) +{ + aiopti_adam_t *opti = (aiopti_adam_t *)(self->optimizer_configuration); + + // beta^t <= beta^t * beta; + *((float *) opti->beta1t) = *((float *) opti->beta1t) * *((float *) opti->beta1); + *((float *) opti->beta2t) = *((float *) opti->beta2t) * *((float *) opti->beta2); + *((float *) opti->one_minus_beta1) = 1.0f - *((float *) opti->beta1); + *((float *) opti->one_minus_beta2) = 1.0f - *((float *) opti->beta2); + + return; +} diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/aiopti/aiopti_adam_default.h b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/aiopti/aiopti_adam_default.h new file mode 100644 index 0000000..8103a3a --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/aiopti/aiopti_adam_default.h @@ -0,0 +1,137 @@ +/** + * \file basic/default/aiopti/aiopti_adam_default.h + * \internal + * \date 28.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link aiopti_adam.h Adam optimizer \endlink + * + * Hardware independent implementations of the Adam optimizer in \link aimath_f32.h F32 \endlink data-type. + * For more information about the Adam optimizer refer to aiopti_adam.h. + */ + +#ifndef AIOPTI_ADAM_DEFAULT +#define AIOPTI_ADAM_DEFAULT + +#include "basic/base/aiopti/aiopti_adam.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct aiopti_adam_f32 aiopti_adam_f32_t; + +/** @brief Data-type specific \link aiopti_adam.h Adam optimizer \endlink struct for \link aimath_f32.h F32 \endlink + * + * Adds data fields for the learning rate and the configuration values in \link aimath_f32.h F32 \endlink to the base implementation. + */ +struct aiopti_adam_f32 { + aiopti_adam_t base; /**< Inherited field members from general optimizer struct. */ + + /** @name Optimizer configuration + * @brief Required configuration parameters for the optimizer + * + * These fields have to be configured by the user before calling the initializer function. + */ + ///@{ + aiscalar_f32_t learning_rate; /**< Storage for aiopti.learning_rate scalar in F32 */ + + aiscalar_f32_t beta1; /**< Storage for aiopti_adam.beta1 scalar in F32 */ + aiscalar_f32_t beta2; /**< Storage for aiopti_adam.beta2 scalar in F32 */ + aiscalar_f32_t eps; /**< Storage for aiopti_adam.eps scalar in F32 */ + ///@} + + /** @name Variables for internal computation + * + * These fields are automatically configured in the initializer function. + */ + ///@{ + aiscalar_f32_t beta1t; /**< Storage for aiopti_adam.beta1t scalar in F32 */ + aiscalar_f32_t beta2t; /**< Storage for aiopti_adam.beta2t scalar in F32 */ + aiscalar_f32_t one_minus_beta1; /**< Storage for aiopti_adam.one_minus_beta1 scalar in F32 */ + aiscalar_f32_t one_minus_beta2; /**< Storage for aiopti_adam.one_minus_beta2 scalar in F32 */ + aiscalar_f32_t lrt; /**< Storage for aiopti_adam.lrt scalar in F32 */ + ///@} +}; + + +/** @brief Initializes an \link aiopti_adam.h Adam optimizer \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * Example: Create the optimizer structure:\n + * \code{.c} + * aiopti_adam_f32_t adam_optimizer = { + * .learning_rate = 0.01f, + * + * .beta1 = 0.9f, + * .beta2 = 0.999f, + * .eps = 1e-7f + * }; + * \endcode + * + * Example: Initialize the optimizer:\n + * \code{.c} + * aiopti_t *optimizer; + * + * optimizer = aiopti_adam_f32_default(&adam_optimizer); + * \endcode + * + * @param *opti The optimizer structure to initialize. + * @return The (successfully) initialized optimizer structure. + */ +aiopti_t *aiopti_adam_f32_default(aiopti_adam_f32_t *opti); + +/** @brief \link aimath_f32.h F32 \endlink default implementation of the aiopti.begin_step function for ADAM + * + * *Implementation of aiopti.begin_step.* + * + * The ADAM optimizer needs some modification of the learning rate in every optimization step. + * This function deals with aiscalars and has to be implemented for every data-type individually. + * + * The calculations are:\n + * @f[ + * lr_t \leftarrow lr \cdot \frac{\sqrt{1 - \beta^t_2}} {(1 - \beta_1)^t} + * @f] + * + * This is not primary time critical function, because it only deals with scalars -> No special hardware implementation necessary (but possible). + * + * @param *self The optimizer structure + */ +void aiopti_adam_f32_default_begin_step(aiopti_t *self); + +/** @brief \link aimath_f32.h F32 \endlink default implementation of the aiopti.end_step function for ADAM + * + * *Implementation of aiopti.end_step.* + * + * The ADAM optimizer needs some modification of the learning rate in every optimization step. + * This function deals with aiscalars and has to be implemented for every data-type individually. + * + * The calculations are:\n + * @f[ + * \beta^t_1 \leftarrow \beta^t_1 \cdot \beta_1 + * @f] + * + * @f[ + * \beta^t_2 \leftarrow \beta^t_2 \cdot \beta_2 + * @f] + * + * This is not primary time critical function, because it only deals with scalars -> No special hardware implementation necessary (but possible). + * + * @param *self The optimizer structure + */ +void aiopti_adam_f32_default_end_step(aiopti_t *self); + +#endif // AIOPTI_ADAM_DEFAULT diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/aiopti/aiopti_sgd_default.c b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/aiopti/aiopti_sgd_default.c new file mode 100644 index 0000000..03ee4e3 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/aiopti/aiopti_sgd_default.c @@ -0,0 +1,58 @@ +/** + * \file basic/default/aiopti/aiopti_sgd_default.c + * \version 2.0alpha + * \date 28.10.2020 + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief + * \details + */ + +#include "basic/default/aiopti/aiopti_sgd_default.h" + +aiopti_t *aiopti_sgd_f32_default(aiopti_sgd_f32_t *opti) +{ + aiopti_t* return_opti; + + opti->base.base.dtype = aif32; + + // Call "constructor" of base "class" + return_opti = aiopti_sgd(&opti->base); + + return_opti->learning_rate = &(opti->learning_rate); + opti->base.momentum = &opti->momentum; + + // Check if a momentum is set and configure the right functions + if (opti->momentum != 0.0f) { + return_opti->update_params = aiopti_sgd_update_params_with_momentum; + return_opti->sizeof_optimem = aiopti_sgd_sizeof_optimem_with_momentum; + return_opti->init_optimem = aiopti_sgd_init_optimem_with_momentum; + } + else { + return_opti->update_params = aiopti_sgd_update_params_without_momentum; + return_opti->sizeof_optimem = aiopti_sgd_sizeof_optimem_without_momentum; + return_opti->init_optimem = aiopti_sgd_init_optimem_without_momentum; + } + + // Set f32 math functions of sgd optimizer + opti->base.zero_tensor = aimath_f32_default_zero_tensor; + opti->base.tensor_add = aimath_f32_default_tensor_add; + opti->base.tensor_sub = aimath_f32_default_tensor_sub; + opti->base.scalar_mul = aimath_f32_default_scalar_mul; + + return return_opti; +} diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/aiopti/aiopti_sgd_default.h b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/aiopti/aiopti_sgd_default.h new file mode 100644 index 0000000..94793dd --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/basic/default/aiopti/aiopti_sgd_default.h @@ -0,0 +1,75 @@ +/** + * \file basic/default/aiopti/aiopti_sgd_default.h + * \internal + * \date 28.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief Default implementation of the \link aiopti_sgd.h Stochastic Gradient Descend optimizer \endlink + * + * Hardware independent implementations of the Stochastic Gradient Descend optimizer in \link aimath_f32.h F32 \endlink data-type. + * For more information about the Stochastic Gradient Descend optimizer refer to aiopti_sgd.h. + */ + +#ifndef AIOPTI_SGD_DEFAULT +#define AIOPTI_SGD_DEFAULT + +#include "basic/base/aiopti/aiopti_sgd.h" + +#include "basic/default/aimath/aimath_f32_default.h" + +typedef struct aiopti_sgd_f32 aiopti_sgd_f32_t; /**< New data type name for code reduction. */ + +/** @brief Data-type specific \link aiopti_sgd.h SGD optimizer \endlink struct for \link aimath_f32.h F32 \endlink + * + * Adds data fields for the learning rate and the momentum in \link aimath_f32.h F32 \endlink to the base implementation. + */ +struct aiopti_sgd_f32 { + aiopti_sgd_t base; /**< Inherited field members from general aiopti_sgd struct. */ + + aiscalar_f32_t learning_rate; /**< Storage for aiopti.learning_rate scalar in F32 */ + + aiscalar_f32_t momentum; /**< Storage for aiopti_sgd.momentum scalar in F32 */ +}; + +/** @brief Initializes a \link aiopti_sgd.h SGD optimizer \endlink with the \link aimath_f32.h F32 \endlink default implementation + * + * You can set the momentum to zero to save memory space. + * + * Example: Create the optimizer structure:\n + * \code{.c} + * aiopti_sgd_f32_t sgd_optimizer = { + * .learning_rate = 0.01f, + * + * .momentum = 0.9f + * }; + * \endcode + * + * Example: Initialize the optimizer:\n + * \code{.c} + * aiopti_t *optimizer; + * + * optimizer = aiopti_sgd_f32_default(&sgd_optimizer); + * \endcode + * + * @param *opti The optimizer structure to initialize. + * @return The (successfully) initialized optimizer structure. + */ +aiopti_t *aiopti_sgd_f32_default(aiopti_sgd_f32_t *opti); + +#endif // AIOPTI_SGD_DEFAULT diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/core/aifes_core.h b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/core/aifes_core.h new file mode 100644 index 0000000..202005f --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/core/aifes_core.h @@ -0,0 +1,466 @@ +/** + * \file core/aifes_core.h + * \internal + * \date 20.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief AIfES 2 core interface + * + * This interface defines the functional modules of AIfES 2 like model, layer, loss and optimizer. + * + * The structure of an aimodel and its components can look like in the following example for a simple multilayer perceptron - MLP: + * \image html functional_concept.jpg width=800px + */ + +#ifndef AIFES_CORE +#define AIFES_CORE + +#include "aifes_math.h" + +#define TRUE 1 +#define FALSE 0 + +// ToDo: Make enum from the values +#define AILAYER_RESULT_LOWER_BOUND 0 +#define AILAYER_RESULT_UPPER_BOUND 1 +#define AILAYER_DELTAS_LOWER_BOUND 2 +#define AILAYER_DELTAS_UPPER_BOUND 3 + +typedef struct ailayer ailayer_t; +typedef struct ailoss ailoss_t; +typedef struct aimodel aimodel_t; +typedef struct aiopti aiopti_t; + +typedef struct aicore_layertype aicore_layertype_t; +typedef struct aicore_losstype aicore_losstype_t; +typedef struct aicore_optitype aicore_optitype_t; + + +/** @brief Type indicator of the layer + * + * Every layer should have a constant global variable of this type which indicates the layer type. + * + * Example for the dense layer:\n + * \code{.c} + * const aicore_layertype_t ailayer_dense_type_s = { + * #ifdef AIDEBUG_PRINT_MODULE_SPECS + * .name = "Dense", + * .print_specs = ailayer_dense_print_specs + * #else + * .name = 0, + * .print_specs = 0 + * #endif + * }; + * const aicore_layertype_t *ailayer_dense_type = &ailayer_dense_type_s; + * \endcode + * + * Checks for the layer type can look like this:\n + * \code{.c} + * ailayer_t *example_layer; + * if(example_layer->layer_type == ailayer_dense_type){ + * ... + * } + * \endcode + */ +struct aicore_layertype { + const char *name; /**< Name of the layer type (for example "Dense") */ + + /** @brief Set a function to print specs of the layer (for example size, constants) + * + * This function should only be set in the debug mode when prints are required (to save memory otherwise) + * + * @param self The layer + * @param *print A function for printing (for example printf) + */ + void (*print_specs)(const ailayer_t *self, int (*print)(const char *format, ...)); +}; + +/** @brief Type indicator of the loss to check for the loss type + * + * Every loss should have a constant global variable of this type which indicates the loss type.\n + * Example for the mse loss:\n + * + * \code{.c} + * const aicore_losstype_t ailoss_mse_type_s = { + * #ifdef AIDEBUG_PRINT_MODULE_SPECS + * .name = "Mean Squared Error", + * .print_specs = ailoss_mse_print_spec + * #else + * .name = 0, + * .print_specs = 0 + * #endif + * }; + * const aicore_losstype_t *ailoss_mse_type = &ailoss_mse_type_s; + * \endcode + * + * Checks for the loss type can look like this:\n + * \code{.c} + * ailoss_t *example_loss; + * if(example_loss->loss_type == ailoss_mse_type){ + * ... + * } + * \endcode + */ +struct aicore_losstype { + const char *name; /**< Name of the loss type (for example "Mean Squared Error") */ + + /** @brief Set a function to print specs of the loss + * + * This function should only be set in the debug mode when prints are required (to save memory otherwise) + * + * @param self The loss + * @param *print A function for printing (for example printf) + */ + void (*print_specs)(const ailoss_t *self, int (*print)(const char *format, ...)); +}; + +/** @brief Type indicator of the optimizer to check for the optimizer type + * + * Every optimizer should have a constant global variable of this type which indicates the optimizer type.\n + * Example for the adam optimizer:\n + * + * \code{.c} + * const aicore_optitype_t aiopti_adam_type_s = { + * #ifdef AIDEBUG_PRINT_MODULE_SPECS + * .name = "ADAM", + * .print_specs = aiopti_adam_print_specs + * #else + * .name = 0, + * .print_specs = 0 + * #endif + * }; + * const aicore_optitype_t *aiopti_adam_type = &aiopti_adam_type_s; + * \endcode + * + * Checks for the optimizer type can look like this:\n + * \code{.c} + * aiopti_t *example_optimizer; + * if(example_optimizer->optimizer_type == aiopti_adam_type){ + * ... + * } + * \endcode + */ +struct aicore_optitype { + const char *name; /**< Name of the optimizer type (for example "ADAM") */ + + /** @brief Set a function to print specs of the optimizer + * + * This function should only be set in the debug mode when prints are required (to save memory otherwise) + * + * @param self The optimizer + * @param *print A function for printing (for example printf) + */ + void (*print_specs)(const aiopti_t *self, int (*print)(const char *format, ...)); +}; + +/** @brief AIfES artificial neural network model +* +* \image html aimodel.png width=500px +* +* This structure holds the model structure and compute graph of the ANN. (Refer to aifes_core.h for a structural overview)\n +* A aimodel contains several layers that are connected in a list or graph structure.\n +* When the model should be trained, a loss function is required. Otherwise it can be set to NULL. +*/ +struct aimodel { + ailayer_t *input_layer; /**< Input layer of the model that gets the input data. */ + ailayer_t *output_layer; /**< Output layer of the model. The output of a forwad pass is available in the result tensor of this layer. */ + + uint16_t layer_count; /**< Total number of layers of the model (usually autogenerated). */ + uint16_t trainable_params_count; /**< Total number of trainable parameter tensors */ + + ailoss_t *loss; /**< The loss or cost function of the model (only for training). */ +}; + + +/** @brief AIfES layer interface +* +* \image html ailayer.png width=200px +* +* The interface contains the necessary functions and parameters for inference and training on the model. +* (Refer to aifes_core.h for a structural overview) +* +* The call order of the functions for inference:\n +* \code{.c} +* +* for each layer in the model +* calc_result_shape() +* endfor +* +* for each layer in the model +* forward() +* endfor +* // The result of the inference is now in output_layer.result tensor +* \endcode +* +* The call order of the functions for training:\n +* \code{.c} +* +* for each layer in the model +* calc_result_shape() +* endfor +* +* // If the parameters are not already pretrained, a new parameter memory block can be created +* for each layer in the model +* allocate memory of size sizeof_paramem() +* set_paramem() +* endfor +* init_trainable_parameters_model() // Do some weight initialization +* +* for each layer in the model +* allocate memory of size sizeof_trainmem() +* set_trainmem() +* endfor +* +* init_trainmem_model() // For example set the optimem to zero +* for iterations +* for each batch in dataset +* zero_gradients_model() +* +* for each sample in the batch +* for each layer in the model +* forward() +* endfor +* +* calc_delta() +* for each layer in the model (reverse) +* backward() +* endfor +* endfor +* +* update_params_model() +* endfor +* endfor +* \endcode +*/ +struct ailayer { + const aicore_layertype_t *layer_type; /**< Type of the layer (for example ailayer_dense_type) */ + void *layer_configuration; /**< Layer specific configurations (back-link from abstract layer class to implementation) */ + + /** @name Layer connections + * Defines the model graph. + */ + ///@{ + ailayer_t *input_layer; + //ailayer_t *brother_input_layer; /**< (NOT_IN_USE) Chained list if multiple input layer are present else NULL. */ + + ailayer_t *output_layer; + //ailayer_t *brother_output_layer; /**< (NOT_IN_USE) Chained list if multiple output layer are present else NULL. */ + ///@} + + /** @name Inference and training scheduling order (Not in use yet) + * @brief The scheduler can execute the layers along this path. + */ + ///@{ + ailayer_t *next_scheduled; + ailayer_t *prev_scheduled; + ///@} + + aitensor_t result; /**< The result of the forward function is stored here */ + + /** @brief Result and delta min and max values. + * + * Returns the min/max values if the result can only be in certain value ranges (like sigmoid). + * This may be used e.g. for quantization algorithms. Set to NULL if not in use. + * + * @param self The layer + * @param selector Select the requested bound (AILAYER_RESULT_LOWER_BOUND, AILAYER_RESULT_UPPER_BOUND, AILAYER_DELTA_LOWER_BOUND, AILAYER_DELTA_UPPER_BOUND) + * @param result_bound A scalar of aiscalar type in which the bound value can be written + * @return TRUE if bound available else FALSE + */ + uint8_t (*get_result_bound)(const ailayer_t *self, const uint8_t selector, void *result_bound); + + /** @brief Calculate and write the shape to the result tensor. + * + * Made for easy creation of the model (layers can be connected to each other without worrying about the shapes). + * + * @param self The layer + */ + void (*calc_result_shape)(ailayer_t *self); + + /** @brief Calculate the forward pass and write the result to the result tensor. + * + * @param self The layer + */ + void (*forward)(ailayer_t *self); + + // Maybe for later purpose + //void (*sizeof_infmem)(struct aifes_layer_t *, void *); + //void (*set_infmem)(struct aifes_layer_t *, void *); + + // ------------------ Used for training only: ----------------------- + + aitensor_t deltas; /**< The result of the backward function is stored here. */ + + /** @name Training memory API + * @brief Makes the memory of the trainable params, the gradients and optimizer stuff accessible. + * + * This is used, for example, for the optimiser or debugging purposes. + */ + ///@{ + uint8_t trainable_params_count; /**< Number of trainable parameter tensors. */ + aitensor_t **trainable_params; /**< Array of tensor pointers with length trainable_params_count. */ + aitensor_t **gradients; /**< Array of tensor pointers with length trainable_params_count. */ + void **optimem; /**< Array of memory pointers with length trainable_params_count. */ + ///@} + + /** @brief Calculate the backward pass and write the result to the deltas tensor. + * + * @param self The layer + */ + void (*backward)(ailayer_t *self); + + /** @name Parameter memory + * @brief Calculate the size and set the memory for the parameter. + * + * This memory (for example for weights, bias, ...) will last through all the lifetime of the model.\n + * This is only intended for training when no initial weights are available. If the parameters are already known, + * set the parameter directly to the layer. + */ + ///@{ + uint32_t (*sizeof_paramem)(const ailayer_t *self); /**< Size of required memory (in bytes). */ + void (*set_paramem)(ailayer_t *self, void* memory_ptr); /**< Set and distribute the memory block internally. */ + ///@} + + /** @name Training memory + * @brief Calculate the size and set the working memory for the training + * + * This memory (for example for gradients, momentums, ...) is needed during the whole training process. + * If the training is finished, it can be deleted. + */ + ///@{ + uint32_t (*sizeof_trainmem)(const ailayer_t *self); /**< Size of required memory (in bytes). */ + void (*set_trainmem)(ailayer_t *self, void* memory_ptr); /**< Set and distribute the memory block internally. */ + ///@} +}; + + +/** @brief AIfES loss interface. +* +* \image html ailoss.png width=300px +* +* The interface contains the necessary functions and parameters for loss and error calculation on the target data. +* (Refer to aifes_core.h for a structural overview) +*/ +struct ailoss { + const aicore_losstype_t *loss_type; /**< Type of the loss (for example ailoss_mse_type) */ + void *loss_configuration; /**< Loss specific configurations (back-link from abstract loss class to implementation) */ + + ailayer_t connection_layer; /**< Dummy layer for docking to the layer structure */ + + /** @brief Calculate the loss / cost for the model on the given targets. + * + * @param self The layer + * @param target_data Tensor containing the target data / labels + * @param result Scalar in which the output can be written (aiscalar of same type as layer type). + */ + void (*calc_loss)(ailoss_t *self, const aitensor_t *target_data, void *result); + + /** @brief Calculate the error on the target data and write it to the deltas tensor of connection layer + * + * @param self The layer + * @param target_data Tensor containing the target data / labels + */ + void (*calc_delta)(ailoss_t *self, const aitensor_t *target_data); +}; + + +/** @brief AIfES optimizer interface. +* +* \image html aiopti.png width=300px +* +* The interface contains the necessary functions and parameters for parameter optimizers in backpropagation training. +* (Refer to aifes_core.h for a structural overview)\n +* Optimizers are responsible for updating the trainable parameters of the model with the gradients +* calculated in the backward pass. +* +* The call order of the functions is:\n +* \code{.c} +* allocate memory of size sizeof_optimem() +* init_optimem() +* +* for batch in dataset +* for each trainable parameter tensor in the model +* zero_gradients() +* endfor +* +* forward_model(batch) +* backward_model(batch) +* +* begin_step() +* for each trainable parameter tensor in the model +* update_params() +* endfor +* end_step() +* endfor +* \endcode +*/ +struct aiopti { + const aicore_optitype_t *optimizer_type; /**< Type of the optimizer (for example aiopti_sgd_type) */ + void *optimizer_configuration; /**< Optimizer specific configurations (back-link from abstract aiopti class to implementation) */ + const aimath_dtype_t *dtype; /**< The data-type of the parameter that the optimizer can optimize and the learning rate */ + + void *learning_rate; /**< The learning rate configures the training speed. The learning rate is an aiscalar_t value of given dtype. */ + + /** @brief Calculates the optimization memory size for a trainable parameter tensor. + * + * @param self The layer + * @param params The trainable parameter tensor + */ + uint32_t (*sizeof_optimem)(aiopti_t *self, const aitensor_t *params); + + /** @brief Initialize the optimization memory for a trainable parameter tensor. + * + * @param self The layer + * @param params The trainable parameter tensor + * @param gradients The associated gradients tensor + * @param optimem The associated optimization memory to initialize + */ + void (*init_optimem)(aiopti_t *self, const aitensor_t *params, const aitensor_t *gradients, void *optimem); + + /** @brief Set the gradient tensor to zero. + * + * @param self The layer + * @param params The gradient tensor + */ + void (*zero_gradients)(aiopti_t *self, aitensor_t *gradients); + + /** @brief Called in the beginning of every model optimization step for parameter initialization. + * + * @param self The layer + */ + void (*begin_step)(aiopti_t *self); + + /** @brief Performs an optimization step on the given tensor. + * + * @param self The layer + * @param params The trainable parameter tensor + * @param gradients The associated gradients tensor + * @param optimem The associated optimization memory to initialize + */ + void (*update_params)(aiopti_t *self, aitensor_t *params, const aitensor_t *gradients, void *optimem); + + /** @brief Called in the end of every model optimization step + * + * @param self The layer + */ + void (*end_step)(aiopti_t *self); +}; + + +#endif // AIFES_CORE diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/core/aifes_math.h b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/core/aifes_math.h new file mode 100644 index 0000000..9580f47 --- /dev/null +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/core/aifes_math.h @@ -0,0 +1,107 @@ +/** + * \file core/aifes_math.h + * \internal + * \date 20.10.2020 + * \endinternal + * \version 2.0alpha + * \copyright Copyright (C) 2020-2021 Fraunhofer Institute for Microelectronic Circuits and Systems. + All rights reserved. + + AIfES is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + * + * \brief AIfES 2 math interface + * + * This interface describes the basic structures to perform data type independent math in AIfES. +*/ + +#ifndef AIFES_MATH +#define AIFES_MATH + +#include +#include +#include +#include + + +#define SHAPE_CHECK /** +#include +#include +#include + +#include "aifes.h" + +#include "MNIST_training_data.h" +#include "MNIST_training_data_label.h" +#include "MNIST_test_data.h" +#include "MNIST_test_data_label.h" +#include "MNIST_weights.h" + +#define INPUTS 784 +#define NEURONS_1 10 +#define NEURONS_2 10 +#define OUTPUTS 10 +#define NUM_TRAIN_DATA 30000 +#define NUM_TEST_DATA 10000 + + +void predict_model_acc(float *model_acc, int num_data, int num_output_neurons, float *data_label, float *model_pred){ + /* + Calculates the accuracy of the neural network + */ + + float argmax_target_acc; + uint8_t argmax_target_index; + float argmax_pred_acc; + uint8_t argmax_pred_index; + *model_acc = 0; + + for(int i=0;i argmax_target_acc) { + argmax_target_acc = *((data_label+i*num_output_neurons)+j); + argmax_target_index = j; + } + if(*((model_pred+i*num_output_neurons)+j) > argmax_pred_acc) { + argmax_pred_acc = *((model_pred+i*num_output_neurons)+j); + argmax_pred_index = j; + } + } + if(argmax_target_index == argmax_pred_index){ + (*model_acc)++; + } + } + *model_acc = (*model_acc/num_data)*100; + +} + + +int main(int argc, char *argv[]) +{ + // loop variables + uint32_t i; + uint32_t j; + uint32_t k; + + float training_acc; + float test_acc; + + uint16_t input_shape[] = {NUM_TRAIN_DATA, INPUTS}; // Definition of the input shape + aitensor_t input_tensor; // Creation of the input AIfES tensor + input_tensor.dtype = aif32; // Definition of the used data type, here float with 32 bits, different ones are available + input_tensor.dim = 2; // Dimensions of the tensor, here 2 dimensions, as specified at input_shape + input_tensor.shape = input_shape; // Set the shape of the input_tensor + input_tensor.data = MNIST_training_data; // Assign the input_data array to the tensor. It expects a pointer to the array where the data is stored + + + uint16_t input_shape_test_data[] = {NUM_TEST_DATA, INPUTS}; // Definition of the input shape + aitensor_t input_tensor_test_data; // Creation of the input AIfES tensor + input_tensor_test_data.dtype = aif32; // Definition of the used data type, here float with 32 bits, different ones are available + input_tensor_test_data.dim = 2; // Dimensions of the tensor, here 2 dimensions, as specified at input_shape + input_tensor_test_data.shape = input_shape_test_data; // Set the shape of the input_tensor + input_tensor_test_data.data = MNIST_test_data; // Assign the input_data array to the tensor. It expects a pointer to the array where the data is stored + + + uint16_t target_shape[] = {NUM_TRAIN_DATA, OUTPUTS}; // Definition of the output shape + aitensor_t target_tensor; // Creation of the input AIfES tensor + target_tensor.dtype = aif32; // Definition of the used data type, here float with 32 bits, different ones are available + target_tensor.dim = 2; // Dimensions of the tensor, here 2 dimensions, as specified at input_shape + target_tensor.shape = target_shape; // Set the shape of the input_tensor + target_tensor.data = MNIST_training_data_label; // Assign the target_data array to the tensor. It expects a pointer to the array where the data is stored + + + uint16_t target_shape_test_data[] = {NUM_TEST_DATA, OUTPUTS}; // Definition of the output shape + aitensor_t target_tensor_test_data; // Creation of the input AIfES tensor + target_tensor_test_data.dtype = aif32; // Definition of the used data type, here float with 32 bits, different ones are available + target_tensor_test_data.dim = 2; // Dimensions of the tensor, here 2 dimensions, as specified at input_shape + target_tensor_test_data.shape = target_shape_test_data; // Set the shape of the input_tensor + target_tensor_test_data.data = MNIST_test_data_label; // Assign the target_data array to the tensor. It expects a pointer to the array where the data is stored + + // Tensor for the output data (result after training). + // Same configuration as for the target tensor + float output_data[NUM_TRAIN_DATA][OUTPUTS]; + uint16_t output_shape[] = {NUM_TRAIN_DATA, OUTPUTS}; + aitensor_t output_tensor; + output_tensor.dtype = aif32; + output_tensor.dim = 2; + output_tensor.shape = output_shape; + output_tensor.data = output_data; + + // Tensor for the output test data (result of testing). + // Same configuration as for the target tensor + float output_test_data[NUM_TEST_DATA][OUTPUTS]; + uint16_t output_shape_test_data[] = {NUM_TEST_DATA, OUTPUTS}; + aitensor_t output_tensor_test_data; + output_tensor_test_data.dtype = aif32; + output_tensor_test_data.dim = 2; + output_tensor_test_data.shape = output_shape_test_data; + output_tensor_test_data.data = output_test_data; + + // ---------------------------------- Layer definition --------------------------------------- + + // Input layer + uint16_t input_layer_shape[] = {1, INPUTS}; // Definition of the input layer shape (Must fit to the input tensor) + ailayer_input_t input_layer; // Creation of the AIfES input layer + input_layer.input_dim = 2; // Definition of the input dimension (Must fit to the input tensor) + input_layer.input_shape = input_layer_shape; // Handover of the input layer shape + + // Dense layer (hidden layer_1) + ailayer_dense_t dense_layer_1; // Creation of the AIfES hidden dense layer + dense_layer_1.neurons = NEURONS_1; // Number of neurons + dense_layer_1.weights.data = weights_data_dense_1; // Passing the hidden layer weights + dense_layer_1.bias.data = bias_data_dense_1; // Passing the hidden layer bias weights + + ailayer_relu_t relu_layer_1; // Relu activation function + + // Dense layer (hidden layer_2) + ailayer_dense_t dense_layer_2; // Creation of the AIfES hidden dense layer + dense_layer_2.neurons = NEURONS_2; // Number of neurons + dense_layer_2.weights.data = weights_data_dense_2; // Passing the hidden layer weights + dense_layer_2.bias.data = bias_data_dense_2; // Passing the hidden layer bias weights + + ailayer_relu_t relu_layer_2; // Relu activation function + + // Output dense layer + ailayer_dense_t dense_layer_3; // Creation of the AIfES ouput dense layer + dense_layer_3.neurons = OUTPUTS; // Number of neurons + dense_layer_3.weights.data = weights_data_dense_3; // Passing the output layer weights + dense_layer_3.bias.data = bias_data_dense_3; // Passing the output layer bias weights + + ailayer_softmax_t softmax_layer_3; // Softmax activation function + + + ailoss_crossentropy_t crossentropy_loss; // Loss: Crossentropy + + // --------------------------- Define the structure of the model ---------------------------- + + aimodel_t model; // AIfES model + ailayer_t *x; // Layer object from AIfES, contains the layers + + // Passing the layers to the AIfES model + model.input_layer = ailayer_input_f32_default(&input_layer); + x = ailayer_dense_f32_default(&dense_layer_1, model.input_layer); + x = ailayer_relu_f32_default(&relu_layer_1, x); + x = ailayer_dense_f32_default(&dense_layer_2, x); + x = ailayer_relu_f32_default(&relu_layer_2, x); + x = ailayer_dense_f32_default(&dense_layer_3, x); + x = ailayer_softmax_f32_default(&softmax_layer_3, x); + model.output_layer = x; + + // Add the loss to the AIfES model + model.loss = ailoss_crossentropy_f32_default(&crossentropy_loss, model.output_layer); + + aialgo_compile_model(&model); // Compile the AIfES model + + // -------------------------------- Define the optimizer for training --------------------- + + aiopti_t *optimizer; // Object for the optimizer + + //ADAM optimizer + aiopti_adam_f32_t adam_opti; + adam_opti.learning_rate = 0.001f; + adam_opti.beta1 = 0.9f; + adam_opti.beta2 = 0.999f; + adam_opti.eps = 1e-7; + + // Choose the optimizer + optimizer = aiopti_adam_f32_default(&adam_opti); + + // -------------------------------- Allocate and schedule the working memory for training --------- + + uint32_t memory_size = aialgo_sizeof_training_memory(&model, optimizer); + printf("Required memory for the training (Intermediate results, gradients, optimization memory): %d Byte\n", memory_size); + + void *memory_ptr = malloc(memory_size); + + // Schedule the memory over the model + aialgo_schedule_training_memory(&model, optimizer, memory_ptr, memory_size); + + // Initialize the AIfES model + aialgo_init_model_for_training(&model, optimizer); + + // --------------------------------- Result with imported keras weights ---------------------------------- + + aialgo_inference_model(&model, &input_tensor_test_data, &output_tensor_test_data); + printf("\n"); + printf("Results with imported weights from keras:\n"); + + for (i = 0; i < 10; i++) { + for(k = 0; k < 2; k++) { + if(k==0){ + printf("real output %d:\n",i); + } + else { + printf("calculated output %d:\n",i); + } + for(j = 0; j < 10; j++) { + if(k==0){ + printf("%f",MNIST_test_data_label[i][j]); + printf("\t"); + } + else { + printf("%f",output_test_data[i][j]); + printf("\t"); + } + } + } + printf("\n\n"); + } + + predict_model_acc(&training_acc, NUM_TEST_DATA, OUTPUTS, &MNIST_test_data_label, &output_test_data); + printf("Model accuracy with imported keras weights: %.3f%%\n\n", training_acc); + + // ------------------------------------- Run the training ------------------------------------ + + float loss_training; + uint32_t batch_size = 200; + uint16_t epochs = 50; + uint16_t print_interval = 10; + + printf("\n"); + printf("Start training\n"); + for(i = 0; i < epochs; i++) + { + // One epoch of training. Iterates through the whole data once + aialgo_train_model(&model, &input_tensor, &target_tensor, optimizer, batch_size); + + // Calculate and print loss every print_interval epochs + if(i % print_interval == 0) + { + aialgo_calc_loss_model_f32(&model, &input_tensor, &target_tensor, &loss_training); + + //reduction / normalization of the loss + loss_training = loss_training / (OUTPUTS * NUM_TRAIN_DATA); + + printf("Epoch: "); + printf("%d",i); + printf(" Loss: "); + printf("%f\n",loss_training); + + } + } + printf("Finished training\n\n"); + + // ----------------------------------------- Evaluate the trained model -------------------------- + + + float loss_test; + + // Do the inference after training + aialgo_inference_model(&model, &input_tensor_test_data, &output_tensor_test_data); + + + printf("After training:\n"); + printf("Results:\n"); + + for (i = 0; i < 10; i++) { + for(k = 0; k < 2; k++) { + if(k==0){ + printf("real output %d:\n",i); + } + else { + printf("calculated output %d:\n",i); + } + for(j = 0; j < 10; j++) { + if(k==0){ + printf("%f",MNIST_test_data_label[i][j]); + printf("\t"); + } + else { + printf("%f",output_test_data[i][j]); + printf("\t"); + } + } + } + printf("\n"); + printf("\n"); + } + + predict_model_acc(&test_acc, NUM_TEST_DATA, OUTPUTS, &MNIST_test_data_label, &output_test_data); + printf("Model accuracy after training: %.3f%%\n\n", test_acc); + + aialgo_calc_loss_model_f32(&model, &input_tensor_test_data, &target_tensor_test_data, &loss_test); + + //reduction / normalization of the loss + loss_test = loss_test / (OUTPUTS * NUM_TRAIN_DATA); + printf("Loss: %f\n",loss_test); + + + + free(memory_ptr); + + system("pause"); + + return 0; +} From 7f03fa3eef3c42862d205d4ff317a998169005a3 Mon Sep 17 00:00:00 2001 From: Daniel Konegen Date: Wed, 22 Sep 2021 15:29:02 +0200 Subject: [PATCH 2/5] MNIST examples --- .../2_MNIST_on_PC/2_MNIST_inference_keras/src/MNIST_to_cc.py | 4 ++++ .../2_MNIST_inference_keras/src/MNIST_weights.py | 4 ++++ examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/main.c | 2 ++ .../2_MNIST_on_PC/3_MNIST_retrain_keras/src/MNIST_to_cc.py | 4 ++++ .../2_MNIST_on_PC/3_MNIST_retrain_keras/src/MNIST_weights.py | 4 ++++ examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/main.c | 2 ++ 6 files changed, 20 insertions(+) diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/MNIST_to_cc.py b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/MNIST_to_cc.py index ffbe941..b006148 100644 --- a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/MNIST_to_cc.py +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/MNIST_to_cc.py @@ -1,3 +1,7 @@ +''' +Author: Hahn-Schickard-Gesellschaft für angewandte Forschung e.V., Daniel Konegen + Marcus Rueb +''' + import tensorflow as tf from tensorflow.keras.utils import to_categorical diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/MNIST_weights.py b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/MNIST_weights.py index aa17b49..34db3ab 100644 --- a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/MNIST_weights.py +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/MNIST_weights.py @@ -1,3 +1,7 @@ +''' +Author: Hahn-Schickard-Gesellschaft für angewandte Forschung e.V., Daniel Konegen + Marcus Rueb +''' + import tensorflow as tf from tensorflow.keras.utils import to_categorical diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/main.c b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/main.c index 8376b56..5990933 100644 --- a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/main.c +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/main.c @@ -1,4 +1,6 @@ /* + Author: Hahn-Schickard-Gesellschaft für angewandte Forschung e.V., Daniel Konegen + Marcus Rueb + The sketch shows an example of how a neural network is executed in AIfES after training in Keras. This is a script to execute and show the accuracy of a neural network for the MNIST dataset on a PC. diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/MNIST_to_cc.py b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/MNIST_to_cc.py index fbfe515..6c137a5 100644 --- a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/MNIST_to_cc.py +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/MNIST_to_cc.py @@ -1,3 +1,7 @@ +''' +Author: Hahn-Schickard-Gesellschaft für angewandte Forschung e.V., Daniel Konegen + Marcus Rueb +''' + import tensorflow as tf from tensorflow.keras.utils import to_categorical diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/MNIST_weights.py b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/MNIST_weights.py index 141c6cb..65e6a1d 100644 --- a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/MNIST_weights.py +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/MNIST_weights.py @@ -1,3 +1,7 @@ +''' +Author: Hahn-Schickard-Gesellschaft für angewandte Forschung e.V., Daniel Konegen + Marcus Rueb +''' + import tensorflow as tf from tensorflow.keras.utils import to_categorical diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/main.c b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/main.c index 62e2ff4..0b3d439 100644 --- a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/main.c +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/main.c @@ -1,4 +1,6 @@ /* + Author: Hahn-Schickard-Gesellschaft für angewandte Forschung e.V., Daniel Konegen + Marcus Rueb + The sketch shows an example of how a neural network is retrained in AIfES after training in Keras. Afterwards, the retrained neural network is executed to check the accuracy of the network. This is a script to retrain and execute a neural network for the MNIST dataset on a PC. From a1f6a13533b60d9d7f2d21c918c493aec12dd955 Mon Sep 17 00:00:00 2001 From: Daniel Konegen Date: Wed, 22 Sep 2021 15:31:48 +0200 Subject: [PATCH 3/5] MNIST examples --- examples/2_MNIST_on_PC/1_MNIST_train/src/MNIST_to_cc.py | 4 ++++ examples/2_MNIST_on_PC/1_MNIST_train/src/main.c | 2 ++ 2 files changed, 6 insertions(+) diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/MNIST_to_cc.py b/examples/2_MNIST_on_PC/1_MNIST_train/src/MNIST_to_cc.py index fbfe515..6c137a5 100644 --- a/examples/2_MNIST_on_PC/1_MNIST_train/src/MNIST_to_cc.py +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/MNIST_to_cc.py @@ -1,3 +1,7 @@ +''' +Author: Hahn-Schickard-Gesellschaft für angewandte Forschung e.V., Daniel Konegen + Marcus Rueb +''' + import tensorflow as tf from tensorflow.keras.utils import to_categorical diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/main.c b/examples/2_MNIST_on_PC/1_MNIST_train/src/main.c index efe2d44..2729341 100644 --- a/examples/2_MNIST_on_PC/1_MNIST_train/src/main.c +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/main.c @@ -1,4 +1,6 @@ /* + Author: Hahn-Schickard-Gesellschaft für angewandte Forschung e.V., Daniel Konegen + Marcus Rueb + The sketch shows an example of how a neural network is trained from scratch in AIfES using training data. This is a script to train a neural network for the MNIST dataset on a PC. From ecb4721d93eac06e14359249584f6f1c467085d6 Mon Sep 17 00:00:00 2001 From: Daniel Konegen Date: Wed, 22 Sep 2021 15:50:37 +0200 Subject: [PATCH 4/5] Update --- examples/2_MNIST_on_PC/1_MNIST_train/src/MNIST_to_cc.py | 6 +++--- .../2_MNIST_inference_keras/src/MNIST_to_cc.py | 6 +++--- .../2_MNIST_on_PC/3_MNIST_retrain_keras/src/MNIST_to_cc.py | 6 +++--- examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/main.c | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/MNIST_to_cc.py b/examples/2_MNIST_on_PC/1_MNIST_train/src/MNIST_to_cc.py index 6c137a5..9aafc44 100644 --- a/examples/2_MNIST_on_PC/1_MNIST_train/src/MNIST_to_cc.py +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/MNIST_to_cc.py @@ -6,7 +6,7 @@ from tensorflow.keras.utils import to_categorical -#Load and preprocess the MNIST data set +# Load and preprocess the MNIST data set (x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data() x_train = x_train.astype(float)/255.0 @@ -18,8 +18,8 @@ print(x_train.shape) print(x_test.shape) -NUM_TRAINING_DATA = 30000 #Max 60000 -NUM_TEST_DATA = 10000 #Max 10000 +NUM_TRAINING_DATA = 30000 # Max 60000 +NUM_TEST_DATA = 10000 # Max 10000 def generate_train_data(): ''' diff --git a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/MNIST_to_cc.py b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/MNIST_to_cc.py index b006148..d4a6a6c 100644 --- a/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/MNIST_to_cc.py +++ b/examples/2_MNIST_on_PC/2_MNIST_inference_keras/src/MNIST_to_cc.py @@ -6,7 +6,7 @@ from tensorflow.keras.utils import to_categorical -#Load and preprocess the MNIST data set +# Load and preprocess the MNIST data set (x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data() x_train = x_train.astype(float)/255.0 @@ -18,8 +18,8 @@ print(x_train.shape) print(x_test.shape) -NUM_TRAINING_DATA = 30000 #Max 60000 -NUM_TEST_DATA = 10000 #Max 10000 +NUM_TRAINING_DATA = 30000 # Max 60000 +NUM_TEST_DATA = 10000 # Max 10000 def generate_train_data(): ''' diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/MNIST_to_cc.py b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/MNIST_to_cc.py index 6c137a5..9aafc44 100644 --- a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/MNIST_to_cc.py +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/MNIST_to_cc.py @@ -6,7 +6,7 @@ from tensorflow.keras.utils import to_categorical -#Load and preprocess the MNIST data set +# Load and preprocess the MNIST data set (x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data() x_train = x_train.astype(float)/255.0 @@ -18,8 +18,8 @@ print(x_train.shape) print(x_test.shape) -NUM_TRAINING_DATA = 30000 #Max 60000 -NUM_TEST_DATA = 10000 #Max 10000 +NUM_TRAINING_DATA = 30000 # Max 60000 +NUM_TEST_DATA = 10000 # Max 10000 def generate_train_data(): ''' diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/main.c b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/main.c index 0b3d439..d7be7ec 100644 --- a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/main.c +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/main.c @@ -138,7 +138,7 @@ int main(int argc, char *argv[]) // Dense layer (hidden layer_1) ailayer_dense_t dense_layer_1; // Creation of the AIfES hidden dense layer - dense_layer_1.neurons = NEURONS_1; // Number of neurons + dense_layer_1.neurons = NEURONS_1; // Number of neurons dense_layer_1.weights.data = weights_data_dense_1; // Passing the hidden layer weights dense_layer_1.bias.data = bias_data_dense_1; // Passing the hidden layer bias weights From e5e7035b48cfe789e59d6f1efb05307aa9cec1f5 Mon Sep 17 00:00:00 2001 From: Daniel Konegen Date: Wed, 22 Sep 2021 16:14:44 +0200 Subject: [PATCH 5/5] Update --- examples/2_MNIST_on_PC/1_MNIST_train/src/main.c | 2 +- examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/main.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/2_MNIST_on_PC/1_MNIST_train/src/main.c b/examples/2_MNIST_on_PC/1_MNIST_train/src/main.c index 2729341..9c2a423 100644 --- a/examples/2_MNIST_on_PC/1_MNIST_train/src/main.c +++ b/examples/2_MNIST_on_PC/1_MNIST_train/src/main.c @@ -325,7 +325,7 @@ int main(int argc, char *argv[]) aialgo_calc_loss_model_f32(&model, &input_tensor_test_data, &target_tensor_test_data, &loss_test); //reduction / normalization of the loss - loss_test = loss_test / (OUTPUTS * NUM_TRAIN_DATA); + loss_test = loss_test / (OUTPUTS * NUM_TEST_DATA); printf("Loss: %f\n",loss_test); diff --git a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/main.c b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/main.c index d7be7ec..120d507 100644 --- a/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/main.c +++ b/examples/2_MNIST_on_PC/3_MNIST_retrain_keras/src/main.c @@ -313,7 +313,7 @@ int main(int argc, char *argv[]) aialgo_calc_loss_model_f32(&model, &input_tensor_test_data, &target_tensor_test_data, &loss_test); //reduction / normalization of the loss - loss_test = loss_test / (OUTPUTS * NUM_TRAIN_DATA); + loss_test = loss_test / (OUTPUTS * NUM_TEST_DATA); printf("Loss: %f\n",loss_test);