eqModelTesting.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from sacred import Experiment\n",
    "import numpy as np\n",
    "import random\n",
    "import warnings\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.callbacks import *\n",
    "from tensorflow.python.eager import context\n",
    "from tensorflow.python.framework import ops\n",
    "from tensorflow.keras.layers import *\n",
    "\n",
    "\n",
    "from modules import helper\n",
    "from modules import gcnModels\n",
    "from modules import LASA\n",
    "\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from pyts.approximation import SymbolicFourierApproximation\n",
    "from pyts.approximation import SymbolicAggregateApproximation\n",
    "from sklearn.metrics import mean_squared_error, mean_absolute_error\n",
    "import datetime"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Settings for a single run configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# base data settings\n",
    "seed_value= 1 # seed value for everything (still something makes the run not deterministic atm.)\n",
    "model_chosen= 'main' # ['main,'nofeatures']\n",
    "network_choice = 'network1'  # which of both networks to use\n",
    "modelN = \"transformer\" # 3 models which use cnns\n",
    "random_state_here = 1  # random state for data split\n",
    "iLen = 1000 # length of the time window used for the prediction. 1000 = 10s  \n",
    "kFolds = 5 # number of k-folds\n",
    "\n",
    "# sfa symbol stuff\n",
    "doFT= True # do the SFA as preprocessing\n",
    "nbins= 6 # number of SFA symbols\n",
    "ncoef= 125 # number of coefs used from the SFA. This also is afterwards the size of the last dim times the number of features\n",
    "\n",
    "\n",
    "# mode selection\n",
    "useSaves= False # use already trained models rather than train again\n",
    "doAbstract= True # can only be used with the transformer to apply LASA\n",
    "doMixedLasa= False # switch dimensions and apply lasa again\n",
    "doStations= True # Only works with the transformer\n",
    "useEmbed= False # currently always false and a mapping is used as embedding\n",
    "\n",
    "# only needed if lasa is active\n",
    "steps= ['max'] # Which steps to try out for LASA. ['max', 'sum'] #only those two are supported\n",
    "layerCombis= ['hl'] # ['lh', 'hl'] as alternative for LASA combination steps\n",
    "localAvgThresholds= [[1.0,1.0]] # thresholds. We suggest just using one combi to run fast. Always have the same value for both thresholds because the second is not supported\n",
    "\n",
    "\n",
    "# model settings:\n",
    "cnnKernal=25\n",
    "gcnNodes= 32\n",
    "finalLayer= 256\n",
    "filters = [16, 32]\n",
    "patience= 15\n",
    "epochs=200\n",
    "header= 6 # number of transformer heads\n",
    "numOfAttentionLayers= 2 # number of transformer layers\n",
    "dffFaktor= 0.5 # dff size in ralation to dmodel, e.g. if dmodel = 16 and dffFaktor = 0.5, dff is 8\n",
    "transDropout= 0.0 # transformer dropout. Not suggested.\n",
    "lastDropout= 0.4 # dropout at the end of the model\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# some helper methods:\n",
    "def print_time():\n",
    "    parser = datetime.datetime.now() \n",
    "    return parser.strftime(\"%d-%m-%Y %H:%M:%S\")  \n",
    "\n",
    "def normalize(inputs): \n",
    "    normalized = []\n",
    "    for eq in inputs:\n",
    "        maks = np.max(np.abs(eq))\n",
    "        if maks != 0:\n",
    "            normalized.append(eq/maks)\n",
    "        else:\n",
    "            normalized.append(eq)\n",
    "    return np.array(normalized)      \n",
    "\n",
    "def targets_to_list(targets): \n",
    "    targets = targets.transpose(2,0,1)\n",
    "\n",
    "    targetList = []\n",
    "    for i in range(0, len(targets)):\n",
    "        targetList.append(targets[i,:,:])\n",
    "        \n",
    "    return targetList\n",
    "\n",
    "\n",
    "\n",
    "def k_fold_split(inputs, targets, seed): \n",
    "\n",
    "    # make sure everything is seeded\n",
    "    import os\n",
    "    os.environ['PYTHONHASHSEED']=str(seed)\n",
    "    import random\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    np.random.permutation(seed)\n",
    "    tf.random.set_seed(seed)\n",
    "    \n",
    "    p = np.random.permutation(len(targets))\n",
    "    \n",
    "    print('min of p = ',np.array(p)[50:100].min())\n",
    "    print('max of p = ',np.array(p)[50:100].max())\n",
    "    print('mean of p = ',np.array(p)[50:100].mean())\n",
    "    inputs = inputs[p]\n",
    "    targets = targets[p]\n",
    "\n",
    "    \n",
    "    ind = int(len(inputs)/5)\n",
    "    inputsK = []\n",
    "    targetsK = []\n",
    "\n",
    "    for i in range(0,5-1):\n",
    "        inputsK.append(inputs[i*ind:(i+1)*ind])\n",
    "        targetsK.append(targets[i*ind:(i+1)*ind])\n",
    "\n",
    "    \n",
    "    inputsK.append(inputs[(i+1)*ind:])\n",
    "    targetsK.append(targets[(i+1)*ind:])\n",
    "  \n",
    "    \n",
    "    return inputsK, targetsK\n",
    "        \n",
    "def merge_splits(inputs, targets, k): # houden\n",
    "    if k != 0:\n",
    "        z=0\n",
    "        inputsTrain = inputs[z]\n",
    "        targetsTrain = targets[z]\n",
    "    else:\n",
    "        z=1\n",
    "        inputsTrain = inputs[z]\n",
    "        targetsTrain = targets[z]\n",
    "\n",
    "    for i in range(z+1, 5):\n",
    "        if i != k:\n",
    "            inputsTrain = np.concatenate((inputsTrain, inputs[i]))\n",
    "            targetsTrain = np.concatenate((targetsTrain, targets[i]))\n",
    "    \n",
    "    return inputsTrain, targetsTrain, inputs[k], targets[k]\n",
    "\n",
    "# SFA preprocessing and map data to [-1, 1] base on the relative position\n",
    "def trans(val, tDict) -> float:\n",
    "    return tDict[val]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Do some inits like the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "        model_chosen = model_chosen #sys.argv[2]\n",
    "\n",
    "        # set seeds\n",
    "        seed= seed_value \n",
    "        seed_value = seed_value\n",
    "        os.environ['PYTHONHASHSEED']=str(seed_value)\n",
    "        random.seed(seed_value)\n",
    "        tf.random.set_seed(seed_value)\n",
    "        np.random.RandomState(seed_value)\n",
    "\n",
    "        np.random.seed(seed_value)\n",
    "\n",
    "        context.set_global_seed(seed_value)\n",
    "        ops.get_default_graph().seed = seed_value\n",
    "\n",
    "        #pip install tensorflow-determinism needed\n",
    "        os.environ['TF_DETERMINISTIC_OPS'] = '1'\n",
    "        os.environ['TF_CUDNN_DETERMINISTIC'] = '1'\n",
    "        np.random.seed(seed_value)\n",
    "\n",
    "        # create early stop\n",
    "        es = tf.keras.callbacks.EarlyStopping(patience=patience, verbose=0, min_delta=0.001, monitor='val_loss', mode='min',baseline=None, restore_best_weights=True)\n",
    "        \n",
    "\n",
    "        #init gpu\n",
    "        physical_devices = tf.config.list_physical_devices('GPU') \n",
    "        for gpu_instance in physical_devices: \n",
    "            tf.config.experimental.set_memory_growth(gpu_instance, True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "        network_choice = network_choice\n",
    "        random_state_here = random_state_here\n",
    "        nbins = nbins\n",
    "        doFT = doFT\n",
    "        ncoef = ncoef\n",
    "\n",
    "        # create test and train data based on dataset and the given parameters\n",
    "        if network_choice == 'network1':\n",
    "            test_set_size = 0.2\n",
    "            inputs = np.load('data/inputs_ci.npy', allow_pickle = True)\n",
    "            targets = np.load('data/targets.npy', allow_pickle = True)\n",
    "            \n",
    "            graph_input = np.load('data/minmax_normalized_laplacian.npy', allow_pickle=True)\n",
    "            graph_input = np.array([graph_input] * inputs.shape[0])\n",
    "\n",
    "            graph_features = np.load('data/station_coords.npy', allow_pickle=True)\n",
    "            graph_features = np.array([graph_features] * inputs.shape[0])\n",
    "\n",
    "        if network_choice == 'network2':\n",
    "            test_set_size = 0.2\n",
    "            inputs = np.load('data/othernetwork/inputs_cw.npy', allow_pickle = True)\n",
    "            targets = np.load('data/othernetwork/targets.npy', allow_pickle = True)\n",
    "            \n",
    "            graph_input = np.load('data/othernetwork/minmax_normalized_laplacian.npy', allow_pickle=True)\n",
    "            \n",
    "            graph_input = np.array([graph_input] * inputs.shape[0])\n",
    "\n",
    "            graph_features = np.load('data/othernetwork/station_coords.npy', allow_pickle=True)\n",
    "            graph_features = np.array([graph_features] * inputs.shape[0])\n",
    "\n",
    "        train_inputs, test_inputs, traingraphinput , testgraphinput, train_graphfeature, test_graphfeature, train_targets, testTargets = train_test_split(inputs,graph_input, graph_features, targets, test_size=test_set_size, random_state=random_state_here)\n",
    "\n",
    "        test_inputs = test_inputs[:, :, :iLen, :]\n",
    "        train_inputs = train_inputs[:, :, :iLen, :]\n",
    "\n",
    "        iLen = iLen \n",
    "\n",
    "\n",
    "        if doFT:\n",
    "            transformerSS = []\n",
    "            sax = SymbolicAggregateApproximation(n_bins=nbins, strategy='uniform')\n",
    "\n",
    "            vocab = sax._check_params(nbins)\n",
    "            tDict = dict()\n",
    "            for i in range(len(vocab)):\n",
    "\n",
    "                halfSize = (len(vocab)-1)/2\n",
    "                tDict[vocab[i]] =float((i - halfSize) / halfSize)\n",
    "\n",
    "            X_new = []\n",
    "            for i in range(train_inputs.shape[-1]):\n",
    "                transformerS = SymbolicFourierApproximation(n_coefs=ncoef,n_bins=nbins, strategy='uniform')\n",
    "                transformerSS.append(transformerS)\n",
    "                transformerS.fit(train_inputs[:,:,:,i].reshape(-1,train_inputs.shape[-2]))\n",
    "                t = transformerS.transform(train_inputs[:,:,:,i].reshape(-1,train_inputs.shape[-2]))\n",
    "\n",
    "                t1 = t\n",
    "                t11 = []\n",
    "                for v in t1: \n",
    "                    t111 = []\n",
    "                    for v2 in v:\n",
    "                        t111.append(trans(v2, tDict))\n",
    "                    t11.append(t111)\n",
    "                tx =t11\n",
    "                print(tx[0][0])\n",
    "\n",
    "                X_new.append(np.expand_dims(tx, axis=2))\n",
    "            X_new = np.concatenate(X_new, axis=2)\n",
    "            X_new = X_new.reshape((train_inputs.shape[0],train_inputs.shape[1],ncoef, train_inputs.shape[-1]))\n",
    "            X_new.shape\n",
    "            train_inputs = X_new\n",
    "\n",
    "\n",
    "\n",
    "            X_new = []\n",
    "            for i in range(test_inputs.shape[-1]):\n",
    "                t = transformerSS[i].transform(test_inputs[:,:,:,i].reshape(-1,test_inputs.shape[-2]))\n",
    "\n",
    "                t2 = t\n",
    "                t22 = []\n",
    "                for v in t2: \n",
    "                    t222 = []\n",
    "                    for v2 in v:\n",
    "                        t222.append(trans(v2, tDict))\n",
    "                    t22.append(t222)\n",
    "                tx = t22\n",
    "                print(tx[0][0])\n",
    "\n",
    "\n",
    "                X_new.append(np.expand_dims(tx, axis=2))\n",
    "            X_new1 = np.concatenate(X_new, axis=2)\n",
    "            X_new1 = X_new1.reshape((test_inputs.shape[0],test_inputs.shape[1],ncoef, test_inputs.shape[-1]))\n",
    "            X_new1.shape\n",
    "            testInputs = X_new1\n",
    "        else:\n",
    "            testInputs =  normalize(test_inputs)\n",
    "        inputsK, targetsK = k_fold_split(train_inputs, train_targets, seed_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def modelTrainChain(combi_list_s, mse_list, rmse_list, mae_list, k, tempName, modelN, doAbstract, doStations, localAvgThresholds, layerCombis, steps, useEmbed,firstRun,useSaves,epochs, header,numOfAttentionLayers,dffFaktor,transDropout,lastDropout,gcnNodes,finalLayer,filters,cnnKernal,trainInputs,train_graphinput,train_graphfeatureinput,trainTargets,valInputs,val_graphinput,val_graphfeatureinput,valTargets,testInputs,doPosEnc=False, batches=20, doMixedLasa=False): \n",
    "\n",
    "            \n",
    "            # reshape the data as needed and create the model based on the data\n",
    "            if modelN == 'transformer':\n",
    "                if doStations:\n",
    "                    print('newShapes')\n",
    "                    print(trainInputs.shape)\n",
    "                    valInputs = np.transpose(valInputs, [0,1,3,2])\n",
    "                    trainInputs = np.transpose(trainInputs, [0,1,3,2])\n",
    "\n",
    "                    if firstRun:\n",
    "                        firstRun = False\n",
    "                        testInputs = np.transpose(testInputs, [0,1,3,2])\n",
    "                        testInputs = np.reshape(testInputs, (testInputs.shape[0],testInputs.shape[1], testInputs.shape[2] * testInputs.shape[3]))\n",
    "\n",
    "                    print(trainInputs.shape)\n",
    "                    valInputs = np.reshape(valInputs, (valInputs.shape[0],valInputs.shape[1], valInputs.shape[2] * valInputs.shape[3]))\n",
    "                    trainInputs = np.reshape(trainInputs, (trainInputs.shape[0],trainInputs.shape[1], trainInputs.shape[2] * trainInputs.shape[3]))\n",
    "                    print(trainInputs.shape)\n",
    "                    model = gcnModels.buildTransModel(valInputs, doStations, seed, useEmbed, model_chosen, header, numOfAttentionLayers, dffFaktor, transDropout, lastDropout, gcnNodes, finalLayer, doPosEnc=doPosEnc)\n",
    "                else:\n",
    "                    print('newShapes')\n",
    "                    print(trainInputs.shape)\n",
    "                    valInputs = np.transpose(valInputs, [0,2,3,1])\n",
    "                    trainInputs = np.transpose(trainInputs, [0,2,3,1])\n",
    "\n",
    "                    if firstRun:\n",
    "                        firstRun = False\n",
    "                        testInputs = np.transpose(testInputs, [0,2,3,1])\n",
    "                        testInputs = np.reshape(testInputs, (testInputs.shape[0],testInputs.shape[1], testInputs.shape[2] * testInputs.shape[3]))\n",
    "\n",
    "                    print(trainInputs.shape)\n",
    "                    valInputs = np.reshape(valInputs, (valInputs.shape[0],valInputs.shape[1], valInputs.shape[2] * valInputs.shape[3]))\n",
    "                    trainInputs = np.reshape(trainInputs, (trainInputs.shape[0],trainInputs.shape[1], trainInputs.shape[2] * trainInputs.shape[3]))\n",
    "                    print(trainInputs.shape)\n",
    "                    model = gcnModels.buildTransModel(valInputs, doStations, seed, useEmbed, model_chosen, header, numOfAttentionLayers, dffFaktor, transDropout, lastDropout, gcnNodes, finalLayer, doPosEnc=False)\n",
    "            elif modelN == 'old':\n",
    "                model = gcnModels.build_old_model(valInputs[0].shape, seed, filters=filters, kernal_size=cnnKernal, finalDense=finalLayer)\n",
    "            elif modelN == 'bloem':\n",
    "                model = gcnModels.build_bloem_model(valInputs[0].shape, seed, filters=filters, kernal_size=cnnKernal, GCNNodes=gcnNodes, finalDense=finalLayer)\n",
    "\n",
    "            # create checkpoint and train or load model!\n",
    "            iteration_checkpoint = tf.keras.callbacks.ModelCheckpoint(\n",
    "                tempName,\n",
    "                monitor='val_loss',\n",
    "                verbose=0,\n",
    "                save_best_only=True\n",
    "            )\n",
    "            print(tempName)\n",
    "            print(model.summary())\n",
    "\n",
    "            if (os.path.isfile('saves/test'+str(k)+'.index') and useSaves):\n",
    "                print('found weights to load! Won\\'t train model!')\n",
    "                model.load_weights('saves/test'+str(k))\n",
    "            else:\n",
    "                history = model.fit(x=[trainInputs,train_graphinput,train_graphfeatureinput], \n",
    "                                    y=targets_to_list(trainTargets),\n",
    "                        epochs=epochs, batch_size=batches,\n",
    "                        validation_data=([valInputs,val_graphinput,val_graphfeatureinput], targets_to_list(valTargets)),verbose=1,callbacks=[es,iteration_checkpoint])#\n",
    "            \n",
    "            # predict model on test data and calculate the scores\n",
    "            print()\n",
    "            print('Fold number:' + str(k))\n",
    "            predictions = model.predict([testInputs,testgraphinput, test_graphfeature])\n",
    "\n",
    "            new_predictions = np.array(predictions)\n",
    "            new_predictions = np.swapaxes(new_predictions,0,2)\n",
    "            new_predictions = np.swapaxes(new_predictions,0,1)\n",
    "            \n",
    "            MSE = []\n",
    "            for i in range(0,5):\n",
    "                MSE.append(mean_squared_error(testTargets[:,:,i], new_predictions[:,:,i]))\n",
    "            print('mse = ',np.array(MSE).mean())\n",
    "            MSE = np.array(MSE).mean()\n",
    "            \n",
    "            RMSE = []\n",
    "            for i in range(0,5):\n",
    "                RMSE.append(mean_squared_error(testTargets[:,:,i], new_predictions[:,:,i], squared=False))\n",
    "            print('rmse = ',np.array(RMSE).mean())\n",
    "            RMSE = np.array(RMSE).mean()\n",
    "            \n",
    "            MAE = []\n",
    "            for i in range(0,5):\n",
    "                MAE.append(mean_absolute_error(testTargets[:,:,i], new_predictions[:,:,i]))\n",
    "            print('MAE = ',np.array(MAE).mean())\n",
    "            MAE = np.array(MAE).mean()\n",
    "            \n",
    "\n",
    "            mse_list.append(MSE)\n",
    "            rmse_list.append(RMSE)\n",
    "            mae_list.append(MAE)\n",
    "\n",
    "            tf.keras.backend.clear_session()\n",
    "\n",
    "            # do lasa\n",
    "            if doAbstract:\n",
    "\n",
    "                earlyPredictorZ = tf.keras.Model(model.inputs, model.layers[2].output)\n",
    "                doMax = False\n",
    "                takeAvg = True\n",
    "\n",
    "                # threshold and step loop with each combi\n",
    "                for thresholdSet in localAvgThresholds:\n",
    "                    for order in layerCombis:\n",
    "                        for step1 in steps:\n",
    "                            for step2 in steps:\n",
    "                                for step3 in steps:\n",
    "\n",
    "\n",
    "                                    k = order+step1+step2+step3+str(thresholdSet[0])+','+str(thresholdSet[1])\n",
    "\n",
    "                                    if(k not in combi_list_s.keys()):\n",
    "                                        d = dict()\n",
    "                                        d['mse'] = []\n",
    "                                        d['rmse'] = []\n",
    "                                        d['mae'] = []\n",
    "                                        d['reduction'] = []\n",
    "                                        if doMixedLasa:\n",
    "                                            d['mse lasa seq'] = []\n",
    "                                            d['rmse lasa seq'] = []\n",
    "                                            d['mae lasa seq'] = []\n",
    "                                            d['double lasa'] = dict()\n",
    "                                        combi_list_s[k] = d\n",
    "                                    else:\n",
    "                                        d = combi_list_s[k]\n",
    "                \n",
    "                                    # new abstracted data\n",
    "                                    newTrain, trainReduction, skipCounterTrain = LASA.abstractDataS([trainInputs,train_graphinput,train_graphfeatureinput] , earlyPredictorZ, order, step1, step2, step3, doMax, thresholdSet, takeAvg = takeAvg, heatLayer = 0, interpolate=False, useEmbed = False, doFidelity=False)\n",
    "                                    newVal, valReduction, skipCounterVal = LASA.abstractDataS([valInputs,val_graphinput,val_graphfeatureinput], earlyPredictorZ, order, step1, step2, step3, doMax, thresholdSet, takeAvg = takeAvg, heatLayer = 0, interpolate=False, useEmbed = False, doFidelity=False)\n",
    "                                    newTest, testReduction, skipCounterTest = LASA.abstractDataS([testInputs,testgraphinput, test_graphfeature] , earlyPredictorZ, order, step1, step2, step3, doMax, thresholdSet, takeAvg = takeAvg, heatLayer = 0, interpolate=False, useEmbed = False, doFidelity=False)\n",
    "\n",
    "                                    # build new model for test and train it and evaluate it\n",
    "                                    model2 = gcnModels.buildTransModel(newVal, doStations, seed, useEmbed, model_chosen, header, numOfAttentionLayers, dffFaktor, transDropout, lastDropout, gcnNodes, finalLayer, doPosEnc=False)\n",
    "\n",
    "                                    tempName2 = tempName + k +'.h5'\n",
    "                                    iteration_checkpoint = tf.keras.callbacks.ModelCheckpoint(\n",
    "                                        tempName2,\n",
    "                                        monitor='val_loss',\n",
    "                                        verbose=0,\n",
    "                                        save_best_only=True\n",
    "                                    )\n",
    "\n",
    "                                    print(model2.summary())\n",
    "\n",
    "                                    history = model2.fit(x=[newTrain,train_graphinput,train_graphfeatureinput], \n",
    "                                                        y=targets_to_list(trainTargets),\n",
    "                                            epochs=epochs, batch_size=20,\n",
    "                                            validation_data=([newVal,val_graphinput,val_graphfeatureinput], targets_to_list(valTargets)),verbose=1,callbacks=[es,iteration_checkpoint])#\n",
    "\n",
    "                                    print()\n",
    "                                    print('total number of epochs ran = ',len(history.history['loss']))\n",
    "                                    print('Fold number:' + str(k))\n",
    "                                    predictions = model2.predict([newTest,testgraphinput, test_graphfeature])\n",
    "\n",
    "                                    new_predictions = np.array(predictions)\n",
    "                                    new_predictions = np.swapaxes(new_predictions,0,2)\n",
    "                                    new_predictions = np.swapaxes(new_predictions,0,1)\n",
    "\n",
    "                                    MSE = []\n",
    "                                    for i in range(0,5):\n",
    "                                        MSE.append(mean_squared_error(testTargets[:,:,i], new_predictions[:,:,i]))\n",
    "                                    print('mse = ',np.array(MSE).mean())\n",
    "                                    MSE = np.array(MSE).mean()\n",
    "\n",
    "                                    RMSE = []\n",
    "                                    for i in range(0,5):\n",
    "                                        RMSE.append(mean_squared_error(testTargets[:,:,i], new_predictions[:,:,i], squared=False))\n",
    "                                    print('rmse = ',np.array(RMSE).mean())\n",
    "                                    RMSE = np.array(RMSE).mean()\n",
    "\n",
    "                                    MAE = []\n",
    "                                    for i in range(0,5):\n",
    "                                        MAE.append(mean_absolute_error(testTargets[:,:,i], new_predictions[:,:,i]))\n",
    "                                    print('MAE = ',np.array(MAE).mean())\n",
    "                                    MAE = np.array(MAE).mean()\n",
    "\n",
    "                                    d['mse'].append(MSE)\n",
    "                                    d['rmse'].append(RMSE)\n",
    "                                    d['mae'].append(MAE)\n",
    "                                    d['reduction'].append(np.average(testReduction))\n",
    "\n",
    "                                    tf.keras.backend.clear_session()\n",
    "\n",
    "                                    if doMixedLasa:\n",
    "                                        nnewTrain = newTrain.reshape((newTrain.shape[0], newTrain.shape[1], 3, -1))\n",
    "                                        nnewVal = newVal.reshape((newVal.shape[0], newVal.shape[1], 3, -1))\n",
    "                                        nnewTest = newTest.reshape((newTest.shape[0], newTest.shape[1], 3, -1))\n",
    "\n",
    "                                        if doStations:\n",
    "                                            nnewVal = np.transpose(nnewVal, [0,1,3,2])\n",
    "                                            nnewTrain = np.transpose(nnewTrain, [0,1,3,2])\n",
    "                                            nnewTest = np.transpose(nnewTest, [0,1,3,2])\n",
    "                                        else:\n",
    "                                            nnewVal = np.transpose(nnewVal, [0,3,1,2])\n",
    "                                            nnewTrain = np.transpose(nnewTrain, [0,3,1,2])\n",
    "                                            nnewTest = np.transpose(nnewTest, [0,3,1,2])\n",
    "                                        \n",
    "                                        modelTrainChain(d['double lasa'], d['mse lasa seq'], d['rmse lasa seq'], d['mae lasa seq'], k, tempName, modelN, doAbstract, (not doStations), localAvgThresholds, layerCombis, steps, useEmbed,True,useSaves,epochs, header,numOfAttentionLayers,dffFaktor,transDropout,lastDropout,gcnNodes,finalLayer,filters,cnnKernal, nnewTrain,train_graphinput,train_graphfeatureinput,trainTargets,nnewVal,val_graphinput,val_graphfeatureinput,valTargets,nnewTest, doPosEnc=False, batches=20, doMixedLasa=False)\n",
    "\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Main Run Part"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# do some basic checks and name setups\n",
    "print(\"GPU is\", \"available\" if tf.config.list_physical_devices(\"GPU\") else \"NOT AVAILABLE\")\n",
    "print_time()\n",
    "warnings.filterwarnings('ignore')\n",
    "if modelN == 'transformer':   \n",
    "    saveName = f\"./results/results.n{network_choice}.rs{random_state_here}.ft{doFT}.s{nbins}.co{ncoef}.h{header}.aL{numOfAttentionLayers}.df{dffFaktor}.td{transDropout}.ld{lastDropout}.gn{gcnNodes}.fl{finalLayer}.st{doStations}.il{iLen}\"\n",
    "else:\n",
    "    saveName = f\"./results/results.m{modelN}.n{network_choice}.rs{random_state_here}.ft{doFT}.s{nbins}.co{ncoef}.h{header}.f1{filters[0]}.f2{filters[1]}.ks{cnnKernal}.gn{gcnNodes}.fl{finalLayer}.il{iLen}\"\n",
    "\n",
    "fullResults = dict()\n",
    "\n",
    "if os.path.isfile(saveName + '.pkl'):\n",
    "    fullResults[\"Error\"] = \"Already done: \" + saveName\n",
    "    print('Already Done ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~')\n",
    "    print(\"Already done: \" + saveName)\n",
    "\n",
    "else: \n",
    "    mse_list = []\n",
    "    rmse_list = []\n",
    "    mae_list = []\n",
    "    combi_list_s = dict()\n",
    "\n",
    "    firstRun = True\n",
    "\n",
    "\n",
    "    # 5 folds loop\n",
    "    for k in range(0,kFolds):\n",
    "        # prepare data\n",
    "        tf.keras.backend.clear_session()\n",
    "        if modelN == 'transformer':\n",
    "            tempName = f\"./models/f{k}.n{network_choice}.rs{random_state_here}.ft{doFT}.s{nbins}.co{ncoef}.h{header}.aL{numOfAttentionLayers}.df{dffFaktor}.td{transDropout}.ld{lastDropout}.gn{gcnNodes}.fl{finalLayer}.st{doStations}.il{iLen}.h5\"\n",
    "        else:\n",
    "            tempName = f\"./models/m{modelN}.f{k}.n{network_choice}.rs{random_state_here}.ft{doFT}.s{nbins}.co{ncoef}.h{header}.f1{filters[0]}.f2{filters[1]}.ks{cnnKernal}.gn{gcnNodes}.fl{finalLayer}.il{iLen}.h5\"\n",
    "\n",
    "        trainInputsAll, trainTargets, valInputsAll, valTargets = merge_splits(inputsK, targetsK, k)\n",
    "\n",
    "        train_graphinput = traingraphinput[0:trainInputsAll.shape[0],:,:]\n",
    "        train_graphfeatureinput = train_graphfeature[0:trainInputsAll.shape[0],:,:]\n",
    "\n",
    "        val_graphinput = traingraphinput[0:valInputsAll.shape[0],:,:]\n",
    "        val_graphfeatureinput = train_graphfeature[0:valInputsAll.shape[0],:,:]\n",
    "\n",
    "        testInputs = testInputs\n",
    "        if doFT:\n",
    "            trainInputs = trainInputsAll\n",
    "            valInputs = valInputsAll\n",
    "        else:\n",
    "            trainInputs = normalize(trainInputsAll)\n",
    "            valInputs = normalize(valInputsAll)        \n",
    "\n",
    "        modelTrainChain(combi_list_s, mse_list, rmse_list, mae_list, k, tempName, modelN, doAbstract, doStations, localAvgThresholds, layerCombis, steps, useEmbed,firstRun,useSaves,epochs,header,numOfAttentionLayers,dffFaktor,transDropout,lastDropout,gcnNodes,finalLayer,filters,cnnKernal,trainInputs,train_graphinput,train_graphfeatureinput,trainTargets,valInputs,val_graphinput,val_graphfeatureinput,valTargets,testInputs,doPosEnc=False, batches=20, doMixedLasa=doMixedLasa)\n",
    "\n",
    "    print('-')\n",
    "    print('-')\n",
    "    print('-')\n",
    "    print('-')\n",
    "    print('all averages = ')\n",
    "    print('mse score = ',np.array(mse_list).mean())\n",
    "    print('rmse score = ',np.array(rmse_list).mean())\n",
    "    print('mae score = ',np.array(mae_list).mean())\n",
    "\n",
    "    fullResults['mse'] = mse_list\n",
    "    fullResults['rmse'] = rmse_list\n",
    "    fullResults['mae'] = mae_list\n",
    "    fullResults['lasa'] = combi_list_s\n",
    "\n",
    "    print(\"Done done\")\n",
    "\n",
    "    # save results of this experiment run\n",
    "    print(saveName)\n",
    "    helper.save_obj(fullResults, str(saveName))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "transformer",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11 (default, Jul 27 2021, 09:42:29) [MSC v.1916 64 bit (AMD64)]"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "65c5abed4221865104e95e72e8e91f212d4a3f321b26a0538caaab09898873da"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}