diff --git a/README.md b/README.md index bc7a54a..89ea6b5 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ TIMECOP ![alt text](https://raw.githubusercontent.com/BBVA/timecop/master/doc/img/timecop.png) -TIMECOP is a RESTful webservice engine that evaluates univariate and multivariate timeseries. It considerates that the time series has 3 stages: the current state of the time series as the last five points, the past state as all the previous points before the current state and the future state as the forecast of the next 5 points. +TIMECOP is a RESTful webservice engine that evaluates univariate and multivariate timeseries. It considerates that the time series has 3 stages: the current state of the time series as the last five points, the past state as all the previous points before the current state and the future state as the forecast of the next steps(custom number). The aim of TIMECOP is to get insight on the behavior of the time series. To achieve this, the engine compares several time series forecasting algorithms and select the best one according to the MAE (mean absolute error) metric. The 3 different algorithms that compounds the engine are: Holt-Winters, ARIMA, and Recurrent Neural Networks using LSTM cells. diff --git a/engines/__init__.pyc b/engines/__init__.pyc deleted file mode 100644 index 6e0e002..0000000 Binary files a/engines/__init__.pyc and /dev/null differ diff --git a/engines/__pycache__/__init__.cpython-36.pyc b/engines/__pycache__/__init__.cpython-36.pyc new file mode 100644 index 0000000..07b148d Binary files /dev/null and b/engines/__pycache__/__init__.cpython-36.pyc differ diff --git a/engines/__pycache__/auto_arima.cpython-36.pyc b/engines/__pycache__/auto_arima.cpython-36.pyc new file mode 100644 index 0000000..09df198 Binary files /dev/null and b/engines/__pycache__/auto_arima.cpython-36.pyc differ diff --git a/engines/__pycache__/functions_timeseries.cpython-36.pyc b/engines/__pycache__/functions_timeseries.cpython-36.pyc new file mode 100644 index 0000000..d39a7cb Binary files /dev/null and b/engines/__pycache__/functions_timeseries.cpython-36.pyc differ diff --git a/engines/__pycache__/helpers.cpython-36.pyc b/engines/__pycache__/helpers.cpython-36.pyc new file mode 100644 index 0000000..ad9bb3f Binary files /dev/null and b/engines/__pycache__/helpers.cpython-36.pyc differ diff --git a/engines/__pycache__/holtwinter.cpython-36.pyc b/engines/__pycache__/holtwinter.cpython-36.pyc new file mode 100644 index 0000000..2a2ee5c Binary files /dev/null and b/engines/__pycache__/holtwinter.cpython-36.pyc differ diff --git a/engines/__pycache__/lstm.cpython-36.pyc b/engines/__pycache__/lstm.cpython-36.pyc new file mode 100644 index 0000000..652f30c Binary files /dev/null and b/engines/__pycache__/lstm.cpython-36.pyc differ diff --git a/engines/__pycache__/var.cpython-36.pyc b/engines/__pycache__/var.cpython-36.pyc new file mode 100644 index 0000000..bd81305 Binary files /dev/null and b/engines/__pycache__/var.cpython-36.pyc differ diff --git a/engines/auto_arima.py b/engines/auto_arima.py index fbbf3e9..d173630 100644 --- a/engines/auto_arima.py +++ b/engines/auto_arima.py @@ -2,15 +2,23 @@ import pandas as pd from pyramid.arima import auto_arima from sklearn.metrics import mean_squared_error,mean_absolute_error -import helpers as h +from . helpers import create_train_test -def anomaly_AutoArima(lista_datos,desv_mse=0): +def anomaly_AutoArima(lista_datos,num_fut,orig_size,desv_mse=0): - lista_puntos = np.arange(0, len(lista_datos),1) + print (orig_size) + + print ("tamanio original") + start_point= int(orig_size) - 100 + + print ("start point " + str(start_point)) + lista_puntos = np.arange(start_point, orig_size,1) - df, df_train, df_test = h.create_train_test(lista_puntos, lista_datos) + print (lista_puntos) + df, df_train, df_test = create_train_test(lista_puntos, lista_datos) engine_output={} + print ("arranca") stepwise_model = auto_arima(df_train['valores'], start_p=1, start_q=1, max_p=3, max_q=3, m=12, start_P=0, seasonal=True, d=1, D=1, trace=False, approx=False, @@ -73,7 +81,7 @@ def anomaly_AutoArima(lista_datos,desv_mse=0): ############## FORECAST START updated_model = stepwise_model.fit(df['valores']) - forecast = updated_model.predict(n_periods=5) + forecast = updated_model.predict(n_periods=num_fut) engine_output['rmse'] = rmse engine_output['mse'] = mse @@ -84,11 +92,10 @@ def anomaly_AutoArima(lista_datos,desv_mse=0): engine_output['engine']='Autoarima' df_future= pd.DataFrame(forecast,columns=['value']) df_future['value']=df_future.value.astype("float32") - df_future['step']= np.arange( len(lista_datos),len(lista_datos)+5,1) + df_future['step']= np.arange( len(lista_datos),len(lista_datos)+num_fut,1) engine_output['future'] = df_future.to_dict(orient='record') testing_data = pd.DataFrame(future_forecast_pred,index = df_test.index,columns=['expected value']) testing_data['step']=testing_data.index engine_output['debug'] = testing_data.to_dict(orient='record') return (engine_output) - diff --git a/engines/auto_arima.pyc b/engines/auto_arima.pyc deleted file mode 100644 index 43f9276..0000000 Binary files a/engines/auto_arima.pyc and /dev/null differ diff --git a/engines/functions_timeseries.py b/engines/functions_timeseries.py index 7ef486c..dead0c4 100644 --- a/engines/functions_timeseries.py +++ b/engines/functions_timeseries.py @@ -1,55 +1,98 @@ -from helpers import merge_two_dicts -from var import anomaly_VAR, univariate_anomaly_VAR -from holtwinter import anomaly_holt -from auto_arima import anomaly_AutoArima -from lstm import anomaly_LSTM, anomaly_uni_LSTM +from engines.helpers import merge_two_dicts +from . var import anomaly_VAR, univariate_anomaly_VAR,univariate_forecast_VAR +from . holtwinter import anomaly_holt,forecast_holt +from . auto_arima import anomaly_AutoArima +from . lstm import anomaly_LSTM, anomaly_uni_LSTM +import traceback -def model_univariate(lista_datos,num_fut,desv_mse): + +def model_univariate(lista_datos,num_fut,desv_mse,train,name): engines_output={} debug = {} - try: - engines_output['LSTM'] = anomaly_uni_LSTM(lista_datos,desv_mse) - debug['LSTM'] = engines_output['LSTM']['debug'] - except Exception as e: - print(e) - print ('ERROR: exception executing LSTM univariate') - #try: - #engines_output['arima'] = anomaly_AutoArima(lista_datos,desv_mse) - #debug['arima'] = engines_output['arima']['debug'] - #except Exception as e: - #print(e) - #print ('ERROR: exception executing Autoarima') - try: - engines_output['VAR'] = univariate_anomaly_VAR(lista_datos) - debug['VAR'] = engines_output['VAR']['debug'] - except Exception as e: - print(e) - print ('ERROR: exception executing VAR') - try: - engines_output['Holtwinters'] = anomaly_holt(lista_datos,desv_mse) - debug['Holtwinters'] = engines_output['Holtwinters']['debug'] - except Exception as e: - print(e) - print ('ERROR: exception executing Holtwinters') + if train != 'True': + filename = './models_temp/'+name + with open(filename,'r') as f: + winner = f.read() + f.close() + if winner == 'LSTM': + try: + engines_output['LSTM'] = anomaly_uni_LSTM(lista_datos,num_fut,desv_mse,train) + debug['LSTM'] = engines_output['LSTM']['debug'] + except Exception as e: + print(e) + print ('ERROR: exception executing LSTM univariate') + elif winner == 'VAR': + engines_output['VAR'] = univariate_forecast_VAR(lista_datos,num_fut,name) + debug['VAR'] = engines_output['VAR']['debug'] + elif winner == 'Holtwinters': + engines_output['Holtwinters'] = forecast_holt(lista_datos,num_fut,desv_mse,name) + debug['Holtwinters'] = engines_output['Holtwinters']['debug'] + else: + print ("Error") - best_mae=999999999 - winner='Holtwinters' - print ('The size is: ') - print (len(engines_output)) - for key, value in engines_output.iteritems(): - print (key + " " + str(value['mae'])) + else: + try: + engines_output['LSTM'] = anomaly_uni_LSTM(lista_datos,num_fut,desv_mse,train) + debug['LSTM'] = engines_output['LSTM']['debug'] + except Exception as e: + print(e) + print ('ERROR: exception executing LSTM univariate') - if value['mae'] < best_mae: - best_mae=value['mae'] - winner=key - print(winner) - + #try: + #if (len(lista_datos) > 100): + ##new_length= + #lista_datos_ari=lista_datos[len(lista_datos)-100:] + #engines_output['arima'] = anomaly_AutoArima(lista_datos_ari,num_fut,len(lista_datos),desv_mse) + #debug['arima'] = engines_output['arima']['debug'] + #except Exception as e: + #print(e) + #print ('ERROR: exception executing Autoarima') + + try: + if (train == 'True'): + engines_output['VAR'] = univariate_anomaly_VAR(lista_datos,num_fut,name) + debug['VAR'] = engines_output['VAR']['debug'] + else: + engines_output['VAR'] = univariate_forecast_VAR(lista_datos,num_fut,name) + debug['VAR'] = engines_output['VAR']['debug'] + except Exception as e: + print(e) + print ('ERROR: exception executing VAR') + + try: + if (train == 'True'): + engines_output['Holtwinters'] = anomaly_holt(lista_datos,num_fut,desv_mse,name) + debug['Holtwinters'] = engines_output['Holtwinters']['debug'] + else: + print ("entra en forecast") + engines_output['Holtwinters'] = forecast_holt(lista_datos,num_fut,desv_mse,name) + debug['Holtwinters'] = engines_output['Holtwinters']['debug'] + except Exception as e: + print(e) + print ('ERROR: exception executing Holtwinters') + + + best_mae=999999999 + winner='Holtwinters' + print ('The size is: ') + print (len(engines_output)) + for key, value in engines_output.items(): + print (key + " " + str(value['mae'])) + if value['mae'] < best_mae: + best_mae=value['mae'] + winner=key + print(winner) + + filename = './models_temp/'+name + with open(filename,'w') as f: + f.write(winner) + f.close() - print winner + print (winner) temp= {} temp['debug']=debug @@ -64,25 +107,29 @@ def model_multivariate(list_var,num_fut,desv_mse): debug = {} try: - engines_output['LSTM'] = anomaly_LSTM(list_var,desv_mse) + engines_output['LSTM'] = anomaly_LSTM(list_var,num_fut,desv_mse) debug['LSTM'] = engines_output['LSTM']['debug'] print (engines_output['LSTM']) except Exception as e: print(e) print ('ERROR: exception executing LSTM') + try: - engines_output['VAR'] = anomaly_VAR(list_var) + engines_output['VAR'] = anomaly_VAR(list_var,num_fut) debug['VAR'] = engines_output['VAR']['debug'] print (engines_output['VAR']) except Exception as e: - print(e) - print ('ERROR: exception executing VAR') + print(Exception) + print("type error: " + str(e)) + print(traceback.format_exc()) + print ('ERROR: exception executing VAR') + best_mae=999999999 winner='LSTM' print ('The size is ') print (len(engines_output)) print (debug) - for key, value in engines_output.iteritems(): + for key, value in engines_output.items(): print (key) print(str(value['mae'])) if value['mae'] < best_mae: @@ -90,7 +137,7 @@ def model_multivariate(list_var,num_fut,desv_mse): best_mae=value['mae'] winner=key - print "el ganador es " + winner + print ("el ganador es " + winner) temp= {} temp['debug']=debug return merge_two_dicts(engines_output[winner] , temp) diff --git a/engines/functions_timeseries.pyc b/engines/functions_timeseries.pyc deleted file mode 100644 index 793afbd..0000000 Binary files a/engines/functions_timeseries.pyc and /dev/null differ diff --git a/engines/helpers.pyc b/engines/helpers.pyc deleted file mode 100644 index 7fba436..0000000 Binary files a/engines/helpers.pyc and /dev/null differ diff --git a/engines/holtwinter.py b/engines/holtwinter.py index dbb4aa8..9e92940 100644 --- a/engines/holtwinter.py +++ b/engines/holtwinter.py @@ -1,15 +1,18 @@ + import numpy as np import pandas as pd from sklearn.metrics import mean_squared_error,mean_absolute_error from statsmodels.tsa.api import ExponentialSmoothing -import helpers as h +from . helpers import create_train_test +import pickle + -def anomaly_holt(lista_datos,desv_mse=0): +def anomaly_holt(lista_datos,num_fut,desv_mse=0,name='NA'): lista_puntos = np.arange(0, len(lista_datos),1) - df, df_train, df_test = h.create_train_test(lista_puntos, lista_datos) + df, df_train, df_test = create_train_test(lista_puntos, lista_datos) engine_output={} @@ -61,7 +64,7 @@ def anomaly_holt(lista_datos,desv_mse=0): ##########GRID to find seasonal n_periods mae_period = 99999999 best_period=0 - for period in range(2,20): + for period in range(2,19): print ("Period: " + str(period)) stepwise_model = ExponentialSmoothing(df_train['valores'],seasonal_periods=period ,trend='add', seasonal='add', ) fit_stepwise_model = stepwise_model.fit() @@ -128,11 +131,23 @@ def anomaly_holt(lista_datos,desv_mse=0): min = df_aler_ult['anomaly_score'].min() df_aler_ult['anomaly_score']= ( df_aler_ult['anomaly_score'] - min ) /(max - min) + print ("Anomaly finished. Start forecasting") stepwise_model1 = ExponentialSmoothing(df['valores'],seasonal_periods=best_period , seasonal='add') print ("Pass the training") fit_stepwise_model1 = stepwise_model1.fit() - future_forecast_pred1 = fit_stepwise_model1.forecast(5) + + + #with open('./models_temp/learned_model.pkl','w') as f: + # pickle.dump(results,f) + + filename='./models_temp/learned_model_holt_winters'+name + with open(filename,'w') as f: + f.write(str(best_period)) + f.close() + + + future_forecast_pred1 = fit_stepwise_model1.forecast(num_fut) print ("Pass the forecast") @@ -146,7 +161,7 @@ def anomaly_holt(lista_datos,desv_mse=0): print ("Only for future") df_future= pd.DataFrame(future_forecast_pred1,columns=['value']) df_future['value']=df_future.value.astype("float32") - df_future['step']= np.arange( len(lista_datos),len(lista_datos)+5,1) + df_future['step']= np.arange( len(lista_datos),len(lista_datos)+num_fut,1) engine_output['future'] = df_future.to_dict(orient='record') test_values['step'] = test_values.index print ("debug de Holtwinters") @@ -154,29 +169,160 @@ def anomaly_holt(lista_datos,desv_mse=0): engine_output['debug'] = test_values.to_dict(orient='record') print ("la prediccion es") - print df_future + print (df_future) return engine_output -def forecast_holt(lista_datos, num_fut): + + + + + + + + + + + + + + + + + + + + + + + +def forecast_holt(lista_datos,num_fut,desv_mse=0,name='NA'): lista_puntos = np.arange(0, len(lista_datos),1) - df = pd.DataFrame() - df['puntos'] = lista_puntos - df['valores'] = lista_datos - df.set_index('puntos',inplace=True) + df, df_train, df_test = create_train_test(lista_puntos, lista_datos) - stepwise_model = ExponentialSmoothing(df,seasonal_periods=len(df) , seasonal='add') + engine_output={} + + best_period=0 + + + #stepwise_model = ExponentialSmoothing(df_train['valores'],seasonal_periods=best_period ,trend='add', seasonal='add', ) + #fit_stepwise_model = stepwise_model.fit() + filename='./models_temp/learned_model_holt_winters'+name + with open(filename,'r') as f: + + best_period = int(f.read()) + f.close() + + print("el dato es ") + print (str(best_period)) + stepwise_model = ExponentialSmoothing(df_train['valores'],seasonal_periods=best_period ,trend='add', seasonal='add', ) fit_stepwise_model = stepwise_model.fit() - fit_forecast_pred = fit_stepwise_model.fittedvalues + + future_forecast_pred = fit_stepwise_model.forecast(len(df_test['valores'])) + print (future_forecast_pred.values) + + + + list_test = df_test['valores'].values + mse_test = (future_forecast_pred - list_test) + test_values = pd.DataFrame(future_forecast_pred,index = df_test.index,columns=['expected value']) + + + print(list_test) + + mse = mean_squared_error(future_forecast_pred.values,list_test) + + print('Model_test mean error: {}'.format(mse)) + rmse = np.sqrt(mse) + print('Model_test root error: {}'.format(rmse)) + + mse_abs_test = abs(mse_test) + + df_aler = pd.DataFrame(future_forecast_pred,index = df.index,columns=['expected value']) + df_aler['step'] = df['puntos'] + df_aler['real_value'] = df_test['valores'] + df_aler['mse'] = mse + df_aler['rmse'] = rmse + df_aler['mae'] = mean_absolute_error(list_test, future_forecast_pred) + df_aler['anomaly_score'] = abs(df_aler['expected value'] - df_aler['real_value']) / df_aler['mae'] + df_aler_ult = df_aler[:5] + df_aler_ult = df_aler_ult[(df_aler_ult.index==df_aler.index.max())|(df_aler_ult.index==((df_aler.index.max())-1)) + |(df_aler_ult.index==((df_aler.index.max())-2))|(df_aler_ult.index==((df_aler.index.max())-3)) + |(df_aler_ult.index==((df_aler.index.max())-4))] + if len(df_aler_ult) == 0: + exists_anom_last_5 = 'FALSE' + else: + exists_anom_last_5 = 'TRUE' + + + df_aler = df_aler[(df_aler['anomaly_score']> 2)] + max = df_aler['anomaly_score'].max() + min = df_aler['anomaly_score'].min() + + df_aler['anomaly_score']= ( df_aler['anomaly_score'] - min ) /(max - min) + + max = df_aler_ult['anomaly_score'].max() + min = df_aler_ult['anomaly_score'].min() + + df_aler_ult['anomaly_score']= ( df_aler_ult['anomaly_score'] - min ) /(max - min) + + print ("Anomaly finished. Start forecasting") + stepwise_model1 = ExponentialSmoothing(df['valores'],seasonal_periods=best_period , seasonal='add') + print ("Pass the training") + fit_stepwise_model1 = stepwise_model1.fit() + + + + + future_forecast_pred1 = fit_stepwise_model1.forecast(num_fut) + print ("Pass the forecast") + + + engine_output['rmse'] = rmse + engine_output['mse'] = mse + engine_output['mae'] = mean_absolute_error(list_test, future_forecast_pred) + engine_output['present_status']=exists_anom_last_5 + engine_output['present_alerts']=df_aler_ult.fillna(0).to_dict(orient='record') + engine_output['past']=df_aler.fillna(0).to_dict(orient='record') + engine_output['engine']='Holtwinters' + print ("Only for future") + df_future= pd.DataFrame(future_forecast_pred1,columns=['value']) + df_future['value']=df_future.value.astype("float32") + df_future['step']= np.arange( len(lista_datos),len(lista_datos)+num_fut,1) + engine_output['future'] = df_future.to_dict(orient='record') + test_values['step'] = test_values.index + print ("debug de Holtwinters") + print (test_values) + engine_output['debug'] = test_values.to_dict(orient='record') + + print ("la prediccion es") + print (df_future) + + return engine_output + +#def forecast_holt(lista_datos, num_fut): + + #lista_puntos = np.arange(0, len(lista_datos),1) + + #df = pd.DataFrame() + #df['puntos'] = lista_puntos + #df['valores'] = lista_datos + + #df.set_index('puntos',inplace=True) + + #stepwise_model = ExponentialSmoothing(df,seasonal_periods=len(df) , seasonal='add') + #fit_stepwise_model = stepwise_model.fit() + + #fit_forecast_pred = fit_stepwise_model.fittedvalues - future_forecast_pred = fit_stepwise_model.forecast(num_fut) + #future_forecast_pred = fit_stepwise_model.forecast(num_fut) - df_result = pd.DataFrame({'puntos':future_forecast_pred.index, 'valores':future_forecast_pred.values}) - df_result.set_index('puntos',inplace=True) - return (df_result) + #df_result = pd.DataFrame({'puntos':future_forecast_pred.index, 'valores':future_forecast_pred.values}) + #df_result.set_index('puntos',inplace=True) + #return (df_result) diff --git a/engines/holtwinter.pyc b/engines/holtwinter.pyc deleted file mode 100644 index c28466e..0000000 Binary files a/engines/holtwinter.pyc and /dev/null differ diff --git a/engines/lstm.py b/engines/lstm.py index fa44582..9afe368 100644 --- a/engines/lstm.py +++ b/engines/lstm.py @@ -1,154 +1,377 @@ + +import numpy as np +import pandas as pd +from sklearn.preprocessing import MinMaxScaler +from sklearn.metrics import mean_squared_error,mean_absolute_error +from keras.models import Sequential +from keras.layers.recurrent import LSTM +from keras.layers.core import Dense +import math +#import helpers as h +from keras.layers import Dropout +from keras.layers.normalization import BatchNormalization import numpy as np import pandas as pd from sklearn.preprocessing import MinMaxScaler +from sklearn.model_selection import GridSearchCV from sklearn.metrics import mean_squared_error,mean_absolute_error from keras.models import Sequential from keras.layers.recurrent import LSTM from keras.layers.core import Dense import math -import helpers as h +from matplotlib import pyplot +from numpy.random import seed +seed(69) +from math import sqrt +from numpy import concatenate +import matplotlib.pyplot as plt +from pandas import read_csv +from pandas import DataFrame +from pandas import concat +from sklearn.preprocessing import MinMaxScaler +from sklearn.preprocessing import LabelEncoder +from sklearn.metrics import mean_squared_error +from keras.models import Sequential +from keras.layers import Dense +from keras.layers import LSTM +from keras.layers import Activation, Dropout +from keras.layers.normalization import BatchNormalization +from keras.models import load_model +import pickle + +#import multiprocessing -def moving_test_window_preds(n_future_preds): - ''' n_future_preds - Represents the number of future predictions we want to make - This coincides with the number of windows that we will move forward - on the test data - ''' - preds_moving = [] # Use this to store the prediction made on each test window - moving_test_window = [test_X[0,:].tolist()] # Creating the first test window - moving_test_window = np.array(moving_test_window) # Making it an numpy array - - for i in range(n_future_preds): - preds_one_step = model.predict(moving_test_window) # Note that this is already a scaled prediction so no need to rescale this - preds_moving.append(preds_one_step[0,0]) # get the value from the numpy 2D array and append to predictions - preds_one_step = preds_one_step.reshape(1,1,1) # Reshaping the prediction to 3D array for concatenation with moving test window - moving_test_window = np.concatenate((moving_test_window[:,1:,:], preds_one_step), axis=1) # This is the new moving test window, where the first element from the window has been removed and the prediction has been appended to the end - - preds_moving = scaler.inverse_transform(preds_moving) - - return preds_moving -def anomaly_uni_LSTM(lista_datos,desv_mse=0): +def add_hlayer(model, num_nodes, return_sequences=False): + model.add(LSTM(num_nodes, return_sequences=return_sequences)) + +def define_model(n_nodes, n_hlayers, dropout, input_data, output_shape): + model = Sequential() + if n_hlayers == 1: + model.add(LSTM(output_dim =int(n_nodes), activation='relu', input_shape =(input_data.shape[1], input_data.shape[2]), + return_sequences=False)) + else: + model.add(LSTM(output_dim =int(n_nodes), activation='relu', input_shape =(input_data.shape[1], input_data.shape[2]), + return_sequences=True)) + model.add(Dropout(dropout)) + #print(n_hlayers) + + for i in range(n_hlayers-1): + #print(i) + if i == n_hlayers-2: + add_hlayer(model, n_nodes, return_sequences=False) + model.add(Dropout(dropout)) + model.add(BatchNormalization()) + else: + add_hlayer(model, n_nodes, return_sequences=True) + model.add(Dropout(dropout)) + model.add(BatchNormalization()) + + model.add(Dense(int(n_nodes/2), activation='relu')) + model.add(Dropout(dropout)) + + model.add(Dense(output_dim=int(output_shape))) + + model.compile(loss='mse', optimizer='adam', metrics=['accuracy']) + return model + + +def hyperparameter_opt(list_hlayers, list_n_nodes, n_dropout, input_data, output_shape): + models_dict = {} + for hlayer in list_hlayers: + for nodes in list_n_nodes: + for drop in n_dropout: + model = define_model(nodes, hlayer, drop, input_data, output_shape) + name = 'model_nlayers_{}_nnodes_{}_dropout_{}'.format(hlayer, nodes, drop) + models_dict[name] = model + print(name) + + return models_dict + +def anomaly_uni_LSTM(lista_datos,num_forecast=10,desv_mse=2,train='True'): + temp= pd.DataFrame(lista_datos,columns=['values']) - data_raw = temp.values.astype("float32") - scaler = MinMaxScaler(feature_range = (0, 1)) - dataset = scaler.fit_transform(data_raw) + scaler_x = MinMaxScaler(feature_range =(-1, 1)) + x = np.array(temp) + #pyplot.plot(x, label='pred') + #pyplot.legend() + #pyplot.show() + x = scaler_x.fit_transform(temp) + x = x[:,0] + print ('x',x) + TRAIN_SIZE = 0.7 + train_size = int(len(x) * TRAIN_SIZE) + test_size = len(x) - train_size - print(data_raw) - TRAIN_SIZE = 0.70 + x_train, x_test = x[0:train_size], x[train_size:len(x)] + print ('x_train',x_train) + print ('x_test',x_test) - train_size = int(len(dataset) * TRAIN_SIZE) - test_size = len(dataset) - train_size - train, test = dataset[0:train_size, :], dataset[train_size-2:len(dataset), :] - # Create test and training sets for one-step-ahead regression. window_size = 1 - train_X, train_Y = h.create_dataset(train, window_size) - test_X, test_Y = h.create_dataset(test, window_size) - forecast_X, forecast_Y = h.create_dataset(dataset,window_size) - train_X = np.reshape(train_X, (train_X.shape[0], 1, train_X.shape[1])) - test_X = np.reshape(test_X, (test_X.shape[0], 1, test_X.shape[1])) - forecast_X = np.reshape(forecast_X, (forecast_X.shape[0], 1, forecast_X.shape[1])) + num_fore = num_forecast + 1 - #############new engine LSTM - model = Sequential() - model.add(LSTM(100, input_shape=(train_X.shape[1], train_X.shape[2]))) - model.add(Dense(1)) - model.compile(loss='mse', optimizer='adam') - history = model.fit(train_X, train_Y, epochs=300, batch_size=100, validation_data=(test_X, test_Y), verbose=0, shuffle=False) + win_train_x, win_train_y = [], [] + for i in range(len(x_train) - window_size - 1): + if len(x_train)<(i+num_fore): + break + a = x_train[i:(i + window_size)] + win_train_x.append(a) + win_train_y.append(x_train[i + window_size: i+num_fore]) - yhat = model.predict(test_X) - - print ("estoy") - yhat_inverse = scaler.inverse_transform(yhat.reshape(-1, 1)) - testY_inverse = scaler.inverse_transform(test_Y.reshape(-1, 1)) - print (len(test_X)) - print (len(test_Y)) - lista_puntos = np.arange(train_size, train_size + test_size,1) + win_train_x = np.array(win_train_x) + print ('win_train_x',win_train_x) + print ('shape win_train_x',win_train_x.shape) + win_train_y = np.array(win_train_y) + print ('win_train_y',win_train_y) + print ('shape win_train_y',win_train_y.shape) + + + win_train_x = win_train_x.reshape((win_train_x.shape[0], 1, win_train_x.shape[1])) + print('reshape win_train_x',win_train_x.shape) + new_test_x = x_test.reshape((x_test.shape[0], 1, 1)) + print ('new_test_x',new_test_x) + + + actual_model='' - print (lista_puntos) - testing_data = pd.DataFrame(yhat_inverse,index =lista_puntos,columns=['expected value']) + + ############### hyperparameter finding + + if (train == 'True'): + + ##################neural network###################### + + models_dict = {} + n_hlayers = [1, 2] + n_nodes = [100, 300, 500] + n_dropout = [0, 0.1, 0.15, 0.20] + + #pruebas + #n_hlayers = [1] + #n_nodes = [500] + #n_dropout = [0.15] + + models_dict = hyperparameter_opt(n_hlayers, n_nodes, n_dropout, win_train_x, num_forecast) + + for model in models_dict: + print(model) + print(models_dict[model].summary()) + + print ('Numero de modelos',len(models_dict)) + + #####getting best model + dict_eval_models = {} + dict_mse_models = {} + for model in models_dict: + # print 'fit model {}'.format(model) + try: + seed(69) + name_model = models_dict[model].fit(win_train_x, win_train_y, epochs=25, verbose=0, shuffle=False) + dict_eval_models[model] = name_model + except: + dict_eval_models[model] = 'Error' + + + print(model) + yhat = models_dict[model].predict(new_test_x) + yhat_test = yhat[:,0] - rmse = math.sqrt(mean_squared_error(testY_inverse, yhat_inverse)) - mse=mean_squared_error(testY_inverse, yhat_inverse) - mae = mean_absolute_error(testY_inverse, yhat_inverse) + temp_res= pd.DataFrame(yhat_test,columns=['values']) + temp_res = np.array(temp_res) + y_yhat_inv = scaler_x.inverse_transform(temp_res) + y_yhat_inv= y_yhat_inv[:,0] + temp_x_test= pd.DataFrame(x_test,columns=['values']) + temp_x_test = np.array(temp_x_test) + x_test_inv = scaler_x.inverse_transform(temp_x_test) + mse = (mean_squared_error(x_test_inv, y_yhat_inv)) + rmse = np.sqrt(mse) + mae = mean_absolute_error(x_test_inv, y_yhat_inv) + print ('mse', mse) + print ('rmse', rmse) + print ('mae', mae) + dict_mse_models[model] = mae + #if mae != min(dict_mse_models, key = dict_mse_models.get): + #del dict_mse_models[model] + + best_model = min(dict_mse_models, key = dict_mse_models.get) + + + print('best_model',best_model) + + models_dict[best_model].save('./models_temp/lstm.model') + actual_model= models_dict[best_model] + + else: + actual_model= load_model('./models_temp/lstm.model') + + yhat = actual_model.predict(new_test_x) + print ('yhat',yhat) + + ###################################save the best model + + #model_filename = "./models_temp/lstm_model" + #models_dict[best_model].save(model_filename) + + yhat_test = yhat[:,0] + print ('yhat_test',yhat_test) + + temp_res= pd.DataFrame(yhat_test,columns=['values']) + temp_res = np.array(temp_res) + + y_yhat_inv = scaler_x.inverse_transform(temp_res) + print ('y_yhat_inv',y_yhat_inv) + + y_yhat_inv= y_yhat_inv[:,0] + + temp_x_test= pd.DataFrame(x_test,columns=['values']) + temp_x_test = np.array(temp_x_test) + x_test_inv = scaler_x.inverse_transform(temp_x_test) + x_test_inv= x_test_inv[:,0] + print ('x_test_inv',x_test_inv) + + #pyplot.plot(x_test_inv, label='real') + #pyplot.plot(y_yhat_inv, label='pred') + #pyplot.legend() + #pyplot.show() + + mse = (mean_squared_error(x_test_inv, y_yhat_inv)) + rmse = np.sqrt(mse) + print ('mse', mse) + print ('rmse', rmse) - - print ("pasa") df_aler = pd.DataFrame() - test=scaler.inverse_transform([test_Y]) - - df_aler['real_value'] = test[0] - - df_aler['expected value'] = yhat_inverse - df_aler['step'] = np.arange(0, len(yhat_inverse),1) - df_aler['mae']=mae - df_aler['mse']=mse - df_aler['anomaly_score'] = abs(df_aler['expected value'] - df_aler['real_value']) / df_aler['mae'] - + lista_puntos = np.arange(train_size, train_size + test_size,1) + testing_data = pd.DataFrame(y_yhat_inv,index =lista_puntos,columns=['expected value']) + + #print ('x_test_inv',x_test_inv) + #print ('y_yhat_inv',y_yhat_inv) + + df_aler['real_value'] = x_test_inv + df_aler['expected_value'] = y_yhat_inv + + df_aler['mse'] = mse + df_aler['puntos'] = df_aler.index + df_aler['puntos'] = df_aler['puntos'] + train_size + df_aler.set_index('puntos',inplace=True) + + + df_aler['rmse'] = rmse + mae = mean_absolute_error(y_yhat_inv, x_test_inv) + df_aler['mae'] = mae + + + df_aler['anomaly_score'] = abs(df_aler['expected_value']-df_aler['real_value'])/df_aler['mae'] + #print (df_aler) + + df_aler_ult = df_aler[:5] + df_aler = df_aler[(df_aler['anomaly_score']> desv_mse)] + max_anom = df_aler['anomaly_score'].max() + min_anom = df_aler['anomaly_score'].min() + + df_aler['anomaly_score'] = ( df_aler['anomaly_score'] - min_anom ) /(max_anom - min_anom) + + print ('Anomaly') df_aler_ult = df_aler[:5] df_aler_ult = df_aler_ult[(df_aler_ult.index==df_aler.index.max())|(df_aler_ult.index==((df_aler.index.max())-1)) - |(df_aler_ult.index==((df_aler.index.max())-2))|(df_aler_ult.index==((df_aler.index.max())-3)) - |(df_aler_ult.index==((df_aler.index.max())-4))] + |(df_aler_ult.index==((df_aler.index.max())-2))|(df_aler_ult.index==((df_aler.index.max())-3)) + |(df_aler_ult.index==((df_aler.index.max())-4))] if len(df_aler_ult) == 0: exists_anom_last_5 = 'FALSE' else: exists_anom_last_5 = 'TRUE' - - df_aler = df_aler[(df_aler['anomaly_score']> 2)] + max_ult = df_aler_ult['anomaly_score'].max() + min_ult = df_aler_ult['anomaly_score'].min() - max = df_aler['anomaly_score'].max() - min = df_aler['anomaly_score'].min() - df_aler['anomaly_score']= ( df_aler['anomaly_score'] - min ) /(max - min) - - - max = df_aler_ult['anomaly_score'].max() - min = df_aler_ult['anomaly_score'].min() - - df_aler_ult['anomaly_score']= ( df_aler_ult['anomaly_score'] - min ) /(max - min) + df_aler_ult['anomaly_score']= ( df_aler_ult['anomaly_score'] - min_ult ) /(max_ult - min_ult) + + #print (df_aler_ult) + + + + ################## forecast + win_todo_x, win_todo_y = [], [] + for i in range(len(x) - window_size - 1): + if len(x)<(i+num_fore): + break + a = x[i:(i + window_size)] + win_todo_x.append(a) + win_todo_y.append(x[i + window_size: i+num_fore]) + + win_todo_x = np.array(win_todo_x) + #print ('win_todo_x',win_todo_x) + #print ('shape win_todo_x',win_todo_x.shape) + + win_todo_y = np.array(win_todo_y) + #print ('win_todo_y',win_todo_y) + #print ('shape win_todo_y',win_todo_y.shape) - pred_scaled =model.predict(forecast_X) - pred = scaler.inverse_transform(pred_scaled) + win_todo_x = win_todo_x.reshape((win_todo_x.shape[0], 1, win_todo_x.shape[1])) + #print('reshape win_todo_x',win_todo_x.shape) + + + name_model = actual_model.fit(win_todo_x, win_todo_y, epochs=25, verbose=0, shuffle=False) - print ("el tamano de la preddicion") - print (len(pred)) - print(pred) - print ('prediccion') - engine_output={} + falta_win_todo_x = x[-num_forecast:] + #print ('falta_win_todo_x',falta_win_todo_x) + #print ('shape falta_win_todo_x',falta_win_todo_x.shape) + + falta_win_todo_x = falta_win_todo_x.reshape(falta_win_todo_x.shape[0],1,1) + #print ('x',x) + #print ('falta_win_todo_x',falta_win_todo_x) + yhat_todo = actual_model.predict(falta_win_todo_x) + #print ('yhat_todo',yhat_todo) + #print ('yhat_todo',yhat_todo[-1,:]) + + temp_res= pd.DataFrame(yhat_todo[-1],columns=['values']) + temp_res = np.array(temp_res) + y_fore_inv = scaler_x.inverse_transform(temp_res) + + y_fore_inv= y_fore_inv[:,0] - engine_output['rmse'] = str(math.sqrt(mse)) + #pyplot.plot(y_fore_inv, label='pred') + #pyplot.legend() + #pyplot.show() + + + engine_output={} + + engine_output['rmse'] = int(rmse) engine_output['mse'] = int(mse) engine_output['mae'] = int(mae) engine_output['present_status']=exists_anom_last_5 engine_output['present_alerts']=df_aler_ult.fillna(0).to_dict(orient='record') engine_output['past']=df_aler.fillna(0).to_dict(orient='record') engine_output['engine']='LSTM' - df_future= pd.DataFrame(pred[len(pred) - 5:],columns=['value']) + + df_future= pd.DataFrame(y_fore_inv,columns=['value']) + df_future['value']=df_future.value.astype("float64") - df_future['step']= np.arange( len(lista_datos),len(lista_datos)+5,1) - engine_output['future'] = df_future.to_dict(orient='record') - print ("llegamos hasta aqui") - #testing_data['excepted value'].astype("float64") + df_future['step']= np.arange(len(x),len(x)+len(y_fore_inv),1) + engine_output['future'] = df_future.fillna(0).to_dict(orient='record') + testing_data['step']=testing_data.index - #testing_data.step.astype("float64") - print ("llegamos hasta aqui2") - engine_output['debug'] = testing_data.to_dict(orient='record') - return (engine_output) + engine_output['debug'] = testing_data.fillna(0).to_dict(orient='record') + + return engine_output + + def series_to_supervised(data, n_in=1, n_out=1, dropnan=True): n_vars = 1 if type(data) is list else data.shape[1] @@ -178,96 +401,164 @@ def series_to_supervised(data, n_in=1, n_out=1, dropnan=True): -def anomaly_LSTM(list_var,desv_mse=0): +def anomaly_LSTM(list_var,num_fut=10,desv_mae=2): + df_var = pd.DataFrame() for i in range(len(list_var)): df_var['var_{}'.format(i)] = list_var[i] - print df_var.head(3) - - normalized_df = (df_var-df_var.min())/(df_var.max()-df_var.min()) - print normalized_df.head(3) + #print df_var + temp_var_ult = pd.DataFrame(df_var[df_var.columns[-1]]) + scaler_y = MinMaxScaler(feature_range =(-1, 1)) + y = scaler_y.fit_transform(temp_var_ult) + #print ('y', y) - values = normalized_df.values + scaler_x = MinMaxScaler(feature_range =(-1, 1)) + #x = np.array(df_var) + #print x + x = scaler_x.fit_transform(df_var) + #x = x[:,0] + #print ('x',x) - TRAIN_SIZE = 0.70 - train_size = int(len(values) * TRAIN_SIZE) - test_size = len(values) - train_size - train, test = values[0:train_size, :], values[train_size:len(values), :] + TRAIN_SIZE = 0.7 + train_size = int(len(x) * TRAIN_SIZE) + test_size = len(x) - train_size - train_X, train_y = train[:, :-1], train[:, -1] - test_X, test_y = test[:, :-1], test[:, -1] + x_train, x_test = x[0:train_size], x[train_size:len(x)] + #print ('x_train',x_train) + #print ('x_test',x_test) + #print ('shape x_test',x_test.shape) - train_X = train_X.reshape((train_X.shape[0], 1, train_X.shape[1])) - test_X = test_X.reshape((test_X.shape[0], 1, test_X.shape[1])) - print(train_X.shape, train_y.shape, test_X.shape, test_y.shape) - - - model = Sequential() - model.add(LSTM(30, input_shape=(train_X.shape[1], train_X.shape[2]),return_sequences=True)) - model.add(LSTM(30)) - #model.add((50)) - model.add(Dense(1)) - model.compile(loss='mae', optimizer='adam') - # fit network - history = model.fit(train_X, train_y, epochs=50, batch_size=72, validation_data=(test_X, test_y), shuffle=False) - - yhat = model.predict(test_X) - - - ###################################Desnormalizacion############################################# - y_hat_df = pd.DataFrame() - y_hat_df['yhat'] = yhat[:,0] - - test_y_df = pd.DataFrame() - test_y_df['yhat'] = test_y - - #nos quedamos con la columna inicial la cual predecimos para desnormalizar - ult = df_var[[df_var.columns[-1]]] - ult['yhat'] = ult[df_var.columns[-1]] - ult.drop(columns=[df_var.columns[-1]],inplace=True) - - #pyplot.plot(normalized_df[normalized_df.columns[-1]], label='real') - #pyplot.plot(y_hat_df, label='pred') - #pyplot.legend() - #pyplot.show() - - op1= (ult.max()-ult.min()) - - desnormalize_y_hat_df = (y_hat_df * op1)+ult.min() - - desnormalize_test_y_df = (test_y_df * op1)+ult.min() - - #pyplot.plot(desnormalize_test_y_df, label='real') - #pyplot.plot(desnormalize_y_hat_df, label='pred') + window_size = 1 + num_fore = num_forecast + 1 + + win_train_x, win_train_y = [], [] + for i in range(len(x_train) - window_size - 1): + if len(x_train)<(i+num_fore): + break + a = x_train[i:(i + window_size)] + win_train_x.append(a) + win_train_y.append(x_train[i + window_size: i+num_fore]) + + win_train_x = np.array(win_train_x) + print ('win_train_x',win_train_x) + print ('shape win_train_x',win_train_x.shape) + win_train_y = np.array(win_train_y) + print ('win_train_y',win_train_y) + print ('shape win_train_y',win_train_y.shape) + win_train_y_var_pred = win_train_y[:,:,-1] + print ('win_train_y_var_pred',win_train_y_var_pred) + print ('shape win_train_y_var_pred',win_train_y_var_pred.shape) + + new_test_x = x_test.reshape((x_test.shape[0], 1, x_test.shape[1])) + print ('new_test_x',new_test_x) + print ('shape new_test_x',new_test_x.shape) + + + + ##################neural network###################### + + models_dict = {} + n_hlayers = [1, 2, 3] + n_nodes = [100, 300, 500,700] + n_dropout = [0, 0.1, 0.15, 0.20] + + #pruebas + #n_hlayers = [1] + #n_nodes = [500] + #n_dropout = [0] + + models_dict = hyperparameter_opt(n_hlayers, n_nodes, n_dropout, win_train_x, num_forecast) + + for model in models_dict: + print(model) + print(models_dict[model].summary()) + + print ('Numero de modelos',len(models_dict)) + + #####getting best model + dict_eval_models = {} + for model in models_dict: + #print 'fit model {}'.format(model) + try: + seed(69) + name_model = models_dict[model].fit(win_train_x, win_train_y_var_pred, epochs=25, verbose=0, shuffle=False) + dict_eval_models[model] = name_model + except: + dict_eval_models[model] = 'Error' + + + dict_mse_models = {} + for model in models_dict: + print(model) + yhat = models_dict[model].predict(new_test_x) + yhat_test = yhat[:,0] + + temp_res= pd.DataFrame(yhat_test,columns=['values']) + temp_res = np.array(temp_res) + y_yhat_inv = scaler_y.inverse_transform(temp_res) + y_yhat_inv= y_yhat_inv[:,0] + + x_test_var_pred = x_test[:,-1] + #print ('x_test_var_pred', x_test_var_pred) + temp_x_test= pd.DataFrame(x_test_var_pred,columns=['values']) + temp_x_test = np.array(temp_x_test) + x_test_inv = scaler_y.inverse_transform(temp_x_test) + x_test_inv= x_test_inv[:,0] + + #pyplot.plot(x_test_inv, label='real') + #pyplot.plot(y_yhat_inv, label='pred') + #pyplot.legend() + #pyplot.show() + + mse = (mean_squared_error(x_test_inv, y_yhat_inv)) + rmse = np.sqrt(mse) + mae = mean_absolute_error(y_yhat_inv, x_test_inv) + print ('mse', mse) + print ('rmse', rmse) + print ('mae', mae) + dict_mse_models[model] = mae + + best_model = min(dict_mse_models, key = dict_mse_models.get) + + + print('best_model',best_model) + yhat = models_dict[best_model].predict(new_test_x) + yhat_test = yhat[:,0] + + temp_res= pd.DataFrame(yhat_test,columns=['values']) + temp_res = np.array(temp_res) + y_yhat_inv = scaler_y.inverse_transform(temp_res) + y_yhat_inv= y_yhat_inv[:,0] + + x_test_var_pred = x_test[:,-1] + #print ('x_test_var_pred', x_test_var_pred) + temp_x_test= pd.DataFrame(x_test_var_pred,columns=['values']) + temp_x_test = np.array(temp_x_test) + x_test_inv = scaler_y.inverse_transform(temp_x_test) + x_test_inv= x_test_inv[:,0] + + #pyplot.plot(x_test_inv, label='real') + #pyplot.plot(y_yhat_inv, label='pred') #pyplot.legend() #pyplot.show() + mse = (mean_squared_error(x_test_inv, y_yhat_inv)) + rmse = np.sqrt(mse) + print ('mse', mse) + print ('rmse', rmse) - test_y_list = desnormalize_test_y_df['yhat'].tolist() - yhat_list = desnormalize_y_hat_df['yhat'].tolist() - ################################### Fin Desnormalizacion############################################# - mse = (mean_squared_error(test_y_list, yhat_list)) - rmse = np.sqrt(mse) df_aler = pd.DataFrame() - print 'mse', mse - print 'rmse', rmse - - print ('yhat_list',len(yhat_list)) - print ('test_y_list',len(test_y_list)) - print ('values',len(values)) - print ('train_size',train_size) - print ('test_size',test_size) - lista_puntos = np.arange(train_size, train_size + test_size,1) - testing_data = pd.DataFrame(yhat_list,index =lista_puntos,columns=['expected value']) + testing_data = pd.DataFrame(y_yhat_inv,index =lista_puntos,columns=['expected value']) + #print ('x_test_inv',x_test_inv) + #print ('y_yhat_inv',y_yhat_inv) - - df_aler['real_value'] = test_y_list - df_aler['expected_value'] = yhat_list + df_aler['real_value'] = x_test_inv + df_aler['expected_value'] = y_yhat_inv df_aler['mse'] = mse df_aler['puntos'] = df_aler.index @@ -276,22 +567,21 @@ def anomaly_LSTM(list_var,desv_mse=0): df_aler['rmse'] = rmse - mae = mean_absolute_error(yhat_list, test_y_list) - df_aler['mae'] = mean_absolute_error(yhat_list, test_y_list) + mae = mean_absolute_error(y_yhat_inv, x_test_inv) + df_aler['mae'] = mae df_aler['anomaly_score'] = abs(df_aler['expected_value']-df_aler['real_value'])/df_aler['mae'] - print df_aler + print (df_aler) + df_aler_ult = df_aler[:5] - df_aler = df_aler[(df_aler['anomaly_score']> 2)] + df_aler = df_aler[(df_aler['anomaly_score']> desv_mse)] max_anom = df_aler['anomaly_score'].max() min_anom = df_aler['anomaly_score'].min() - df_aler['anomaly_score']= ( df_aler['anomaly_score'] - min_anom ) /(max_anom - min_anom) + df_aler['anomaly_score'] = ( df_aler['anomaly_score'] - min_anom ) /(max_anom - min_anom) - print ('Anomaly') - - print df_aler + #print ('Anomaly') df_aler_ult = df_aler[:5] df_aler_ult = df_aler_ult[(df_aler_ult.index==df_aler.index.max())|(df_aler_ult.index==((df_aler.index.max())-1)) @@ -305,98 +595,84 @@ def anomaly_LSTM(list_var,desv_mse=0): min_ult = df_aler_ult['anomaly_score'].min() df_aler_ult['anomaly_score']= ( df_aler_ult['anomaly_score'] - min_ult ) /(max_ult - min_ult) - df_aler_ult = df_aler_ult.fillna(0) - ################################### Forecast ############################################# + #print (df_aler_ult) + ###forecast + win_todo_x, win_todo_y = [], [] + for i in range(len(x) - window_size - 1): + if len(x)<(i+num_fore): + break + a = x[i:(i + window_size)] + win_todo_x.append(a) + win_todo_y.append(x[i + window_size: i+num_fore]) + win_todo_x = np.array(win_todo_x) + #print ('win_todo_x',win_todo_x) + #print ('shape win_todo_x',win_todo_x.shape) - test1_X, test1_y = values[:, :-1], values[:, -1] - test1_X = test1_X.reshape((test1_X.shape[0], 1, test1_X.shape[1])) + win_todo_y = np.array(win_todo_y) + #print ('win_todo_y',win_todo_y) + #print ('shape win_todo_y',win_todo_y.shape) - model = Sequential() - model.add(LSTM(50, input_shape=(test1_X.shape[1], test1_X.shape[2]),return_sequences=True)) - model.add(LSTM(50)) - #model.add(LSTM(50)) - model.add(Dense(1)) - model.compile(loss='mae', optimizer='adam') - # fit network - history = model.fit(test1_X, test1_y, epochs=50, batch_size=72, verbose=0, shuffle=False) + win_todo_y_var_pred = win_todo_y[:,:,-1] + #print ('win_todo_y_var_pred',win_todo_y_var_pred) + #print ('shape win_todo_y_var_pred',win_todo_y_var_pred.shape) - num_fut=5 - len_fore = len(test1_X) - num_fut - fore = test1_X[len_fore:] - yhat_fore = model.predict(fore) + name_model = models_dict[best_model].fit(win_todo_x, win_todo_y_var_pred, epochs=25, verbose=0, shuffle=False) + falta_win_todo_x = x[-num_forecast:,:] + #print ('falta_win_todo_x',falta_win_todo_x) + #print ('shape falta_win_todo_x',falta_win_todo_x.shape) + falta_win_todo_x = falta_win_todo_x.reshape(falta_win_todo_x.shape[0],1,falta_win_todo_x.shape[1]) + #print ()'x',x) + #print ('falta_win_todo_x',falta_win_todo_x) - ###################################Desnormalizacion############################################# - y_hat_df_fore = pd.DataFrame() - y_hat_df_fore['yhat'] = yhat_fore[:,0] + yhat_todo = models_dict[best_model].predict(falta_win_todo_x) + #print ('yhat_todo',yhat_todo) + #print ('yhat_todo',yhat_todo[-1,:]) + temp_res= pd.DataFrame(yhat_todo[-1],columns=['values']) + temp_res = np.array(temp_res) + y_fore_inv = scaler_y.inverse_transform(temp_res) + y_fore_inv= y_fore_inv[:,0] - op1= (ult.max()-ult.min()) + #pyplot.plot(y_fore_inv, label='pred') + #pyplot.legend() + #pyplot.show() - desnormalize_y_hat_fore = (y_hat_df_fore * op1)+ult.min() + engine_output={} + engine_output['rmse'] = int(rmse) + engine_output['mse'] = int(mse) + engine_output['mae'] = int(mae) + engine_output['present_status']=exists_anom_last_5 + engine_output['present_alerts']=df_aler_ult.fillna(0).to_dict(orient='record') + engine_output['past']=df_aler.fillna(0).to_dict(orient='record') + engine_output['engine']='LSTM' + df_future= pd.DataFrame(y_fore_inv,columns=['value']) - #pyplot.plot(desnormalize_y_hat_fore, label='pred') - #pyplot.legend() - #pyplot.show() + df_future['value']=df_future.value.astype("float64") + df_future['step']= np.arange(len(x),len(x)+len(y_fore_inv),1) + engine_output['future'] = df_future.to_dict(orient='record') - yhat_fore_list = desnormalize_y_hat_fore['yhat'].tolist() + testing_data['step']=testing_data.index + engine_output['debug'] = testing_data.to_dict(orient='record') + return engine_output - lista_result = np.arange(len(test1_X), (len(test1_X)+num_fut),1) - df_result_forecast = pd.DataFrame({'puntos':lista_result, 'valores':yhat_fore_list}) - df_result_forecast.set_index('puntos',inplace=True) - df_result_forecast['valores']=df_result_forecast['valores'].astype(str) - df_result_forecast['step'] = df_result_forecast.index - engine_output={} - engine_output['rmse'] = rmse - engine_output['mse'] = mse - engine_output['mae'] = mae - engine_output['present_status']=exists_anom_last_5 - engine_output['present_alerts']=df_aler_ult.to_dict(orient='record') - engine_output['past']=df_aler.to_dict(orient='record') - engine_output['engine']='LSTM' - engine_output['future']= df_result_forecast.to_dict(orient='record') - testing_data['step']=testing_data.index - engine_output['debug'] = testing_data.to_dict(orient='record') - return (engine_output) +#list_var=[5938,6925,4353,6855,6216,7061,4417,7505,6778,8169,5290,7710,6213,7952,5476,7990,7747,8908,5369,7794,7543,8533,5564,6997,6678,7730,4790,6027,5877,7413,4459,5966,5305,6238,3689,5146,4799,6044,3695,4795,4147,5254,2679,3428,3635,5391,3142,3986,3807,4732,2513,3834,3133,4405] -def forecast_LSTM(list_var,num_fut): - - df_var = pd.DataFrame() - - for i in range(len(list_var)): - df_var['var_{}'.format(i)] = list_var[i] - - values = df_var.values +#num_fut=10 +#aa = anomaly_uni_LSTM(list_var,num_fut) +#print aa - test_X, test_y = values[:, :-1], values[:, -1] - test_X = test_X.reshape((test_X.shape[0], 1, test_X.shape[1])) - - model = Sequential() - model.add(LSTM(50, input_shape=(test_X.shape[1], test_X.shape[2]))) - model.add(Dense(1)) - model.compile(loss='mae', optimizer='adam') - # fit network - - history = model.fit(test_X, test_y, epochs=50, batch_size=72, verbose=0, shuffle=False) - - len_fore = len(test_X) - num_fut - fore = test_X[len_fore:] - yhat = model.predict(fore) - lista_result = np.arange(len(test_X), (len(test_X)+num_fut),1) - df_result = pd.DataFrame({'puntos':lista_result, 'valores':yhat[:,0]}) - df_result.set_index('puntos',inplace=True) - return (df_result) diff --git a/engines/lstm.pyc b/engines/lstm.pyc deleted file mode 100644 index a955148..0000000 Binary files a/engines/lstm.pyc and /dev/null differ diff --git a/engines/var.py b/engines/var.py index 05b4ac8..3c1fa73 100644 --- a/engines/var.py +++ b/engines/var.py @@ -2,18 +2,18 @@ import pandas as pd from sklearn.metrics import mean_squared_error,mean_absolute_error import pyflux as pf -import helpers as h +#from helpers import helpers as h - -def univariate_anomaly_VAR(lista_datos): +def univariate_anomaly_VAR(lista_datos,num_fut,name): lista_puntos = np.arange(0, len(lista_datos),1) df = pd.DataFrame() df['valores'] = lista_datos + df['valores'] = df.valores.astype(np.float) tam_train = int(len(df)*0.7) #print tam_train @@ -21,32 +21,218 @@ def univariate_anomaly_VAR(lista_datos): print('Tamanio train: {}'.format(df_train.shape)) df_test = df[tam_train:] print('Tamanio test: {}'.format(df_test.shape)) + + print (type(df_test)) + mae_period = 99999999 + best_lag=0 + lags = int(round(len(df_train)/2)) + print ("empezamos el bucle") + for lag in range(lags): + model = pf.VAR(df_train,lags=lag) + x = model.fit() + + + print ("fit ready") + future_forecast_pred = model.predict(len(df_test)) + future_forecast_pred = future_forecast_pred[['valores']] + + list_test = df_test['valores'].values + list_future_forecast_pred = future_forecast_pred['valores'].values - model = pf.VAR(df_train,lags=15) + #pyplot.plot(list_test, label='real') + #pyplot.plot(list_future_forecast_pred, label='pred') + #pyplot.legend() + #pyplot.show() + + mae_temp = mean_absolute_error(list_test, list_future_forecast_pred) + print('El error medio del modelo_test es: {}'.format(mae_temp)) + + if mae_temp < mae_period: + best_lag=lag + mae_period=mae_temp + else: + print ("mae:" + str(mae_period)) + + print ("######best mae is " + str(mae_period) + " with the lag " + str(best_lag)) + + model = pf.VAR(df_train,lags=best_lag) x = model.fit() #model.plot_z(list(range(0,6)),figsize=(15,5)) #model.plot_fit(figsize=(8,5)) #model.plot_predict_is(h=8, figsize=((8,5))) #model.plot_predict(past_values=20, h=6, figsize=(8,5)) - + future_forecast_pred = model.predict(len(df_test)) future_forecast_pred = future_forecast_pred[['valores']] list_test = df_test['valores'].values list_future_forecast_pred = future_forecast_pred['valores'].values - + #mse_test = (list_future_forecast_pred - list_test) #mse_abs_test = abs(mse_test) + + #pyplot.plot(list_test, label='real') + #pyplot.plot(list_future_forecast_pred, label='pred') + #pyplot.legend() + #pyplot.show() + + mse = mean_squared_error(list_test, list_future_forecast_pred) + print('El error medio del modelo_test es: {}'.format(mse)) + + + rmse = np.sqrt(mse) + print('El root error medio del modelo_test es: {}'.format(rmse)) + mae = mean_absolute_error(list_test, list_future_forecast_pred) + + filename = './models_temp/learned_model_var'+name + with open(filename,'w') as f: + f.write(str(best_lag)) + f.close() + + + + df_aler = pd.DataFrame() + df_aler['real_value'] = list_test + df_aler['expected value'] = list_future_forecast_pred + df_aler['mse'] = mse + df_aler['puntos'] = future_forecast_pred.index + df_aler.set_index('puntos',inplace=True) + df_aler['mae'] = mae + + df_aler['anomaly_score'] = abs(df_aler['expected value']-df_aler['real_value'])/df_aler['mae'] + + df_aler = df_aler[(df_aler['anomaly_score']> 2)] + + max = df_aler['anomaly_score'].max() + min = df_aler['anomaly_score'].min() + df_aler['anomaly_score']= ( df_aler['anomaly_score'] - min ) /(max - min) + + df_aler_ult = df_aler[:5] + df_aler_ult = df_aler_ult[(df_aler_ult.index==df_aler.index.max())|(df_aler_ult.index==((df_aler.index.max())-1)) + |(df_aler_ult.index==((df_aler.index.max())-2))|(df_aler_ult.index==((df_aler.index.max())-3)) + |(df_aler_ult.index==((df_aler.index.max())-4))] + if len(df_aler_ult) == 0: + exists_anom_last_5 = 'FALSE' + else: + exists_anom_last_5 = 'TRUE' + + max = df_aler_ult['anomaly_score'].max() + min = df_aler_ult['anomaly_score'].min() + print (df_aler_ult) + df_aler_ult['anomaly_score'] = ( df_aler_ult['anomaly_score'] - min ) /(max - min) + + #####forecast##### + + model_for = pf.VAR(df,lags=best_lag) + x_for = model_for.fit() + + #model.plot_z(list(range(0,6)),figsize=(15,5)) + #model.plot_fit(figsize=(8,5)) + + future_forecast_pred_for = model_for.predict(num_fut) + + #pyplot.plot(future_forecast_pred_for, label='forecast') + #pyplot.legend() + #pyplot.show() + + df_result_forecast = future_forecast_pred_for.reset_index() + df_result_forecast = df_result_forecast.rename(columns = {'index':'step'}) + + print (df.head(5)) + print (df.tail(5)) + + engine_output={} + engine_output['rmse'] = rmse + engine_output['mse'] = mse + engine_output['mae'] = mae + engine_output['present_status']=exists_anom_last_5 + engine_output['present_alerts']=df_aler_ult.fillna(0).to_dict(orient='record') + engine_output['past']=df_aler.to_dict(orient='record') + engine_output['engine']='VAR' + engine_output['future']= df_result_forecast.fillna(0).to_dict(orient='record') + test_values = pd.DataFrame(future_forecast_pred.values,index = df_test.index,columns=['expected value']) + test_values['step'] = test_values.index + engine_output['debug'] = test_values.fillna(0).to_dict(orient='record') + + return (engine_output) + + + + + + + + + + + + + +def univariate_forecast_VAR(lista_datos,num_fut,name): + lista_puntos = np.arange(0, len(lista_datos),1) + + + df = pd.DataFrame() + df['valores'] = lista_datos + + df['valores'] = df.valores.astype(np.float) + + tam_train = int(len(df)*0.7) + #print tam_train + df_train = df[:tam_train] + print('Tamanio train: {}'.format(df_train.shape)) + df_test = df[tam_train:] + print('Tamanio test: {}'.format(df_test.shape)) + + print (type(df_test)) + mae_period = 99999999 + best_lag=0 + lags = int(round(len(df_train)/2)) + + filename = './models_temp/learned_model_var'+name + + with open(filename,'r') as f: + + best_lag = int(f.read()) + f.close() + + + + model = pf.VAR(df_train,lags=best_lag) + x = model.fit() + + #model.plot_z(list(range(0,6)),figsize=(15,5)) + #model.plot_fit(figsize=(8,5)) + #model.plot_predict_is(h=8, figsize=((8,5))) + #model.plot_predict(past_values=20, h=6, figsize=(8,5)) + + future_forecast_pred = model.predict(len(df_test)) + future_forecast_pred = future_forecast_pred[['valores']] + + list_test = df_test['valores'].values + list_future_forecast_pred = future_forecast_pred['valores'].values + + #mse_test = (list_future_forecast_pred - list_test) + #mse_abs_test = abs(mse_test) + + #pyplot.plot(list_test, label='real') + #pyplot.plot(list_future_forecast_pred, label='pred') + #pyplot.legend() + #pyplot.show() + mse = mean_squared_error(list_test, list_future_forecast_pred) print('El error medio del modelo_test es: {}'.format(mse)) + + rmse = np.sqrt(mse) print('El root error medio del modelo_test es: {}'.format(rmse)) mae = mean_absolute_error(list_test, list_future_forecast_pred) + df_aler = pd.DataFrame() df_aler['real_value'] = list_test df_aler['expected value'] = list_future_forecast_pred @@ -74,43 +260,48 @@ def univariate_anomaly_VAR(lista_datos): max = df_aler_ult['anomaly_score'].max() min = df_aler_ult['anomaly_score'].min() - print df_aler_ult + print (df_aler_ult) df_aler_ult['anomaly_score'] = ( df_aler_ult['anomaly_score'] - min ) /(max - min) #####forecast##### - model_for = pf.VAR(df,lags=5) + model_for = pf.VAR(df,lags=best_lag) x_for = model_for.fit() #model.plot_z(list(range(0,6)),figsize=(15,5)) #model.plot_fit(figsize=(8,5)) - future_forecast_pred_for = model_for.predict(5) + future_forecast_pred_for = model_for.predict(num_fut) + + #pyplot.plot(future_forecast_pred_for, label='forecast') + #pyplot.legend() + #pyplot.show() df_result_forecast = future_forecast_pred_for.reset_index() df_result_forecast = df_result_forecast.rename(columns = {'index':'step'}) - print df.head(5) - print df.tail(5) + print (df.head(5)) + print (df.tail(5)) engine_output={} engine_output['rmse'] = rmse engine_output['mse'] = mse engine_output['mae'] = mae engine_output['present_status']=exists_anom_last_5 - engine_output['present_alerts']=df_aler_ult.to_dict(orient='record') + engine_output['present_alerts']=df_aler_ult.fillna(0).to_dict(orient='record') engine_output['past']=df_aler.to_dict(orient='record') engine_output['engine']='VAR' - engine_output['future']= df_result_forecast.to_dict(orient='record') + engine_output['future']= df_result_forecast.fillna(0).to_dict(orient='record') test_values = pd.DataFrame(future_forecast_pred.values,index = df_test.index,columns=['expected value']) test_values['step'] = test_values.index - engine_output['debug'] = test_values.to_dict(orient='record') + engine_output['debug'] = test_values.fillna(0).to_dict(orient='record') return (engine_output) -def anomaly_VAR(list_var): + +def anomaly_VAR(list_var,num_fut): df_var = pd.DataFrame() for i in range(len(list_var)): @@ -126,8 +317,43 @@ def anomaly_VAR(list_var): df_test = df_var[tam_train:] print('Tamanio test: {}'.format(df_test.shape)) - lags = int(round(len(df_test)/2)) - model = pf.VAR(df_train,lags=lags) + + mae_period = 99999999 + best_lag=0 + lags = int(round(len(df_train)/2)) + if (lags > 100): + lags=100 + for lag in range(lags): + print ("entra en el bucle con dato " + str(lag)) + model = pf.VAR(df_train,lags=lag) + x = model.fit() + + + future_forecast_pred = model.predict(len(df_test)) + future_forecast_pred = future_forecast_pred[['expected value']] + + list_test = df_test['expected value'].values + list_future_forecast_pred = future_forecast_pred['expected value'].values + + #pyplot.plot(list_test, label='real') + #pyplot.plot(list_future_forecast_pred, label='pred') + #pyplot.legend() + #pyplot.show() + + mae_temp = mean_absolute_error(list_test, list_future_forecast_pred) + print('El error medio del modelo_test es: {}'.format(mae_temp)) + + if mae_temp < mae_period: + best_lag=lag + mae_period=mae_temp + else: + print ("mae:" + str(mae_period)) + print ("sale del bucle") + + print ("######best mae is " + str(mae_period) + " with the lag " + str(best_lag)) + + + model = pf.VAR(df_train,lags=best_lag) x = model.fit() #model.plot_z(list(range(0,6)),figsize=(15,5)) @@ -135,12 +361,18 @@ def anomaly_VAR(list_var): #model.plot_predict_is(h=90, figsize=((8,5))) #model.plot_predict(past_values=len(df_train), h=len(df_test), figsize=(8,5)) + future_forecast_pred = model.predict(len(df_test)) future_forecast_pred = future_forecast_pred[['expected value']] list_test = df_test['expected value'].values list_future_forecast_pred = future_forecast_pred['expected value'].values + + #pyplot.plot(list_test, label='real') + #pyplot.plot(list_future_forecast_pred, label='pred') + #pyplot.legend() + #pyplot.show() #mse_test = (list_future_forecast_pred - list_test) #mse_abs_test = abs(mse_test) @@ -182,13 +414,19 @@ def anomaly_VAR(list_var): df_aler_ult = df_aler_ult.fillna(0) #####forecast##### - model_for = pf.VAR(df_var,lags=5) + model_for = pf.VAR(df_var,lags=best_lag) x_for = model_for.fit() #model.plot_z(list(range(0,6)),figsize=(15,5)) #model.plot_fit(figsize=(8,5)) - future_forecast_pred_for = model_for.predict(5) + # save the model to disk + filename = "./models_temp/var_model.pkl" + with open(filename, 'wb') as file: + pickle.dump(model, file) + + + future_forecast_pred_for = model_for.predict(num_fut) future_forecast_pred_for = future_forecast_pred_for[['expected value']] df_result_forecast = future_forecast_pred_for.reset_index() @@ -201,21 +439,21 @@ def anomaly_VAR(list_var): engine_output['mse'] = mse engine_output['mae'] = mae engine_output['present_status']=exists_anom_last_5 - engine_output['present_alerts']=df_aler_ult.to_dict(orient='record') + engine_output['present_alerts']=df_aler_ult.fillna(0).to_dict(orient='record') engine_output['past']=df_aler.to_dict(orient='record') engine_output['engine']='VAR' - engine_output['future']= df_result_forecast.to_dict(orient='record') + engine_output['future']= df_result_forecast.fillna(0).to_dict(orient='record') engine_output['rmse'] = rmse engine_output['mse'] = mse engine_output['mae'] = mae engine_output['present_status']=exists_anom_last_5 - engine_output['present_alerts']=df_aler_ult.to_dict(orient='record') - engine_output['past']=df_aler.to_dict(orient='record') + engine_output['present_alerts']=df_aler_ult.fillna(0).to_dict(orient='record') + engine_output['past']=df_aler.fillna(0).to_dict(orient='record') engine_output['engine']='VAR' - engine_output['future']= df_result_forecast.to_dict(orient='record') + engine_output['future']= df_result_forecast.fillna(0).to_dict(orient='record') test_values = pd.DataFrame(future_forecast_pred.values,index = df_test.index,columns=['expected value']) test_values['step'] = test_values.index - engine_output['debug'] = test_values.to_dict(orient='record') + engine_output['debug'] = test_values.fillna(0).to_dict(orient='record') return (engine_output) diff --git a/engines/var.pyc b/engines/var.pyc deleted file mode 100644 index 4da67b1..0000000 Binary files a/engines/var.pyc and /dev/null differ diff --git a/main.py b/main.py new file mode 120000 index 0000000..48c45cb --- /dev/null +++ b/main.py @@ -0,0 +1 @@ +server.py \ No newline at end of file diff --git a/models_temp/learned_model_holt_winterstest b/models_temp/learned_model_holt_winterstest new file mode 100644 index 0000000..e440e5c --- /dev/null +++ b/models_temp/learned_model_holt_winterstest @@ -0,0 +1 @@ +3 \ No newline at end of file diff --git a/models_temp/learned_model_holt_winterstest1 b/models_temp/learned_model_holt_winterstest1 new file mode 100644 index 0000000..3f10ffe --- /dev/null +++ b/models_temp/learned_model_holt_winterstest1 @@ -0,0 +1 @@ +15 \ No newline at end of file diff --git a/models_temp/learned_model_holt_winterstest2 b/models_temp/learned_model_holt_winterstest2 new file mode 100644 index 0000000..bf0d87a --- /dev/null +++ b/models_temp/learned_model_holt_winterstest2 @@ -0,0 +1 @@ +4 \ No newline at end of file diff --git a/models_temp/learned_model_vartest b/models_temp/learned_model_vartest new file mode 100644 index 0000000..5f1a9f3 --- /dev/null +++ b/models_temp/learned_model_vartest @@ -0,0 +1 @@ +324 \ No newline at end of file diff --git a/models_temp/learned_model_vartest1 b/models_temp/learned_model_vartest1 new file mode 100644 index 0000000..615088b --- /dev/null +++ b/models_temp/learned_model_vartest1 @@ -0,0 +1 @@ +108 \ No newline at end of file diff --git a/models_temp/learned_model_vartest2 b/models_temp/learned_model_vartest2 new file mode 100644 index 0000000..b393560 --- /dev/null +++ b/models_temp/learned_model_vartest2 @@ -0,0 +1 @@ +23 \ No newline at end of file diff --git a/models_temp/lstm.model b/models_temp/lstm.model new file mode 100644 index 0000000..c32d357 Binary files /dev/null and b/models_temp/lstm.model differ diff --git a/models_temp/test b/models_temp/test new file mode 100644 index 0000000..2baa689 --- /dev/null +++ b/models_temp/test @@ -0,0 +1 @@ +VAR \ No newline at end of file diff --git a/models_temp/test1 b/models_temp/test1 new file mode 100644 index 0000000..dec0143 --- /dev/null +++ b/models_temp/test1 @@ -0,0 +1 @@ +LSTM \ No newline at end of file diff --git a/models_temp/test2 b/models_temp/test2 new file mode 100644 index 0000000..2baa689 --- /dev/null +++ b/models_temp/test2 @@ -0,0 +1 @@ +VAR \ No newline at end of file diff --git a/prueba.lst b/prueba.lst new file mode 100644 index 0000000..f49f9a7 --- /dev/null +++ b/prueba.lst @@ -0,0 +1 @@ +[1, 3, 5, 3, 2, 1, 5, 6, 7, 8, 9, 10, 11, 9, 4, 5, 6, 7, 98, 9, 1, 2, 3, 4, 65, 7, 8, 89, 0, 0, 0, 2, 3, 4, 5, 6, 7, 8, 1, 3, 5, 3, 2, 1, 5, 6, 7, 8, 9, 10, 11, 9, 4, 5, 6, 7, 98, 9, 1, 2, 3, 4, 65, 7, 8, 89, 0, 0, 0, 2, 3, 4, 5, 6, 7, 8, 1, 3, 5, 3, 2, 1, 5, 6, 7, 8, 9, 10, 11, 9, 4, 5, 6, 7, 98, 9, 1, 2, 3, 4, 65, 7, 8, 89, 0, 0, 0, 2, 3, 4, 5, 6, 7, 8] \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..b1dd646 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +Flask_Cors==3.0.6 +matplotlib==2.2.2 +Keras==2.2.2 +pyflux==0.4.15 +pandas==0.23.3 +Flask==1.0.2 +statsmodels==0.8.0 +pyramid_arima==0.5.1 +pyramid==1.10.1 +scikit_learn==0.20.0 diff --git a/server.py b/server.py index 36e391f..a2ff2fd 100644 --- a/server.py +++ b/server.py @@ -1,5 +1,6 @@ from flask import request, Flask, jsonify, abort from flask_cors import CORS +import json import engines.functions_timeseries as ft @@ -16,10 +17,29 @@ def univariate_engine(): timedata = request.get_json() lista=timedata['data'] - num_fut = 5 - desv_mse = 0 + num_fut = int(timedata.get('num_future', 5)) + desv_mae = int(timedata.get('desv_metric', 2)) + name = timedata.get('name', 'NA') + train = timedata.get('train', 'True') + + + if(name != 'NA'): + filename= './lst/'+name+'.lst' + try: + with open(filename, 'r') as filehandle: + previousList = json.load(filehandle) + except Exception: + previousList=[] + + lista = previousList + lista + with open(filename, 'w') as filehandle: + json.dump(lista,filehandle) + + + #desv_mse = 0 + + salida = ft.model_univariate(lista,num_fut,desv_mae,train,name) - salida = ft.model_univariate(lista,num_fut,desv_mse) return jsonify(salida), 201 @@ -31,18 +51,52 @@ def multivariate_engine(): timedata = request.get_json() items = timedata['timeseries'] + name = timedata.get('name', 'NA') list_var=[] for item in items: data = item['data'] - list_var.append(data) + if(name != 'NA'): + sub_name = item['name'] + + filename= './lst/'+name + '_' + sub_name +'.lst' + try: + with open(filename, 'r') as filehandle: + previousList = json.load(filehandle) + except Exception: + previousList=[] + + lista = previousList + data + with open(filename, 'w') as filehandle: + json.dump(lista,filehandle) + - list_var.append(timedata['main']) + list_var.append(data) + + + + lista = timedata['main'] + if(name != 'NA'): + filename= './lst/'+name+'.lst' + try: + with open(filename, 'r') as filehandle: + previousList = json.load(filehandle) + except Exception: + previousList=[] + + lista = previousList + lista + with open(filename, 'w') as filehandle: + json.dump(lista,filehandle) - num_fut = 5 + list_var.append(lista) + + num_fut = int(timedata.get('num_future', 5)) + desv_mae = int(timedata.get('desv_metric', 2)) + + desv_mse = 0 - salida = ft.model_multivariate(list_var,num_fut,desv_mse) - print(salida) + salida = ft.model_multivariate(list_var,num_fut,desv_mae) + #print(salida) return jsonify(salida), 201 @@ -51,4 +105,4 @@ def index(): return "Timecop ready to play" if __name__ == '__main__': - app.run(debug=True)#,host = '0.0.0.0') + app.run(debug=True,host = '0.0.0.0',port=3000) diff --git a/web_test/_arrivals_from_australia_monthly.csv b/web_test/_arrivals_from_australia_monthly.csv new file mode 100644 index 0000000..b490e14 --- /dev/null +++ b/web_test/_arrivals_from_australia_monthly.csv @@ -0,0 +1,129 @@ +Date,Number of arrivals +1991M01,27566 +1991M02,27621 +1991M03,25696 +1991M04,21653 +1991M05,21197 +1991M06,21620 +1991M07,25596 +1991M08,28327 +1991M09,29892 +1991M10,28206 +1991M11,28718 +1991M12,44288 +1992M01,29219 +1992M02,29644 +1992M03,32218 +1992M04,29586 +1992M05,22089 +1992M06,28209 +1992M07,33675 +1992M08,25075 +1992M09,32186 +1992M10,27235 +1992M11,29864 +1992M12,49103 +1993M01,29164 +1993M02,29603 +1993M03,32186 +1993M04,24415 +1993M05,19784 +1993M06,24414 +1993M07,27565 +1993M08,27195 +1993M09,34042 +1993M10,37434 +1993M11,35694 +1993M12,48706 +1994M01,31852 +1994M02,30595 +1994M03,34674 +1994M04,27604 +1994M05,22198 +1994M06,26123 +1994M07,28663 +1994M08,27622 +1994M09,40449 +1994M10,29890 +1994M11,28611 +1994M12,52268 +1995M01,33107 +1995M02,31596 +1995M03,38201 +1995M04,36443 +1995M05,23401 +1995M06,25007 +1995M07,30838 +1995M08,28133 +1995M09,40717 +1995M10,30156 +1995M11,31128 +1995M12,53754 +1996M01,35064 +1996M02,35736 +1996M03,39570 +1996M04,38185 +1996M05,24885 +1996M06,31983 +1996M07,31330 +1996M08,31692 +1996M09,41228 +1996M10,35142 +1996M11,36248 +1996M12,58774 +1997M01,36981 +1997M02,36434 +1997M03,41582 +1997M04,33090 +1997M05,27913 +1997M06,30197 +1997M07,32034 +1997M08,33434 +1997M09,41170 +1997M10,34119 +1997M11,35007 +1997M12,54617 +1998M01,39892 +1998M02,41970 +1998M03,41204 +1998M04,46232 +1998M05,31122 +1998M06,31839 +1998M07,40017 +1998M08,37335 +1998M09,46586 +1998M10,40656 +1998M11,43900 +1998M12,61656 +1999M01,41678 +1999M02,41267 +1999M03,46116 +1999M04,44875 +1999M05,32437 +1999M06,32732 +1999M07,41276 +1999M08,40579 +1999M09,49177 +1999M10,42140 +1999M11,44589 +1999M12,69672 +2000M01,46057 +2000M02,49286 +2000M03,51877 +2000M04,41966 +2000M05,33160 +2000M06,34671 +2000M07,44117 +2000M08,45356 +2000M09,51756 +2000M10,46904 +2000M11,48200 +2000M12,78352 +2001M01,53264 +2001M02,51909 +2001M03,58021 +2001M04,56304 +2001M05,39324 +2001M06,43422 +2001M07,49671 +2001M08,55260 diff --git a/web_test/index.html b/web_test/production/index.html similarity index 82% rename from web_test/index.html rename to web_test/production/index.html index 5da73e2..6e349ba 100644 --- a/web_test/index.html +++ b/web_test/production/index.html @@ -1 +1 @@ -
{{errorDialog.text}}\r\n
{{errorDialog.text}}\r\n
{{convertedText}}\r\n\r\n\r\n\r\n\n\n\n// WEBPACK FOOTER //\n// src/view/converter.vue","var render = function () {var _vm=this;var _h=_vm.$createElement;var _c=_vm._self._c||_h;return _c('pre',[_vm._v(_vm._s(_vm.convertedText))])}\nvar staticRenderFns = []\nvar esExports = { render: render, staticRenderFns: staticRenderFns }\nexport default esExports\n\n\n//////////////////\n// WEBPACK FOOTER\n// ./node_modules/vue-loader/lib/template-compiler?{\"id\":\"data-v-511afa56\",\"hasScoped\":false,\"transformToRequire\":{\"video\":[\"src\",\"poster\"],\"source\":\"src\",\"img\":\"src\",\"image\":\"xlink:href\"},\"buble\":{\"transforms\":{}}}!./node_modules/vue-loader/lib/selector.js?type=template&index=0!./src/view/converter.vue\n// module id = null\n// module chunks = ","var normalizeComponent = require(\"!../../node_modules/vue-loader/lib/component-normalizer\")\n/* script */\nexport * from \"!!babel-loader!../../node_modules/vue-loader/lib/selector?type=script&index=0!./converter.vue\"\nimport __vue_script__ from \"!!babel-loader!../../node_modules/vue-loader/lib/selector?type=script&index=0!./converter.vue\"\n/* template */\nimport __vue_template__ from \"!!../../node_modules/vue-loader/lib/template-compiler/index?{\\\"id\\\":\\\"data-v-511afa56\\\",\\\"hasScoped\\\":false,\\\"transformToRequire\\\":{\\\"video\\\":[\\\"src\\\",\\\"poster\\\"],\\\"source\\\":\\\"src\\\",\\\"img\\\":\\\"src\\\",\\\"image\\\":\\\"xlink:href\\\"},\\\"buble\\\":{\\\"transforms\\\":{}}}!../../node_modules/vue-loader/lib/selector?type=template&index=0!./converter.vue\"\n/* template functional */\nvar __vue_template_functional__ = false\n/* styles */\nvar __vue_styles__ = null\n/* scopeId */\nvar __vue_scopeId__ = null\n/* moduleIdentifier (server only) */\nvar __vue_module_identifier__ = null\nvar Component = normalizeComponent(\n __vue_script__,\n __vue_template__,\n __vue_template_functional__,\n __vue_styles__,\n __vue_scopeId__,\n __vue_module_identifier__\n)\n\nexport default Component.exports\n\n\n\n//////////////////\n// WEBPACK FOOTER\n// ./src/view/converter.vue\n// module id = null\n// module chunks = ","import Vue from 'vue'\nimport Router from 'vue-router'\nimport home from '@/view/homeView'\nimport converter from '@/view/converter'\n\nVue.use(Router)\n\nexport default new Router({\n routes: [\n {\n path: '/',\n name: 'home',\n component: home\n },\n {\n path: '/converter',\n name: 'converter',\n component: converter\n }\n ]\n})\n\n\n\n// WEBPACK FOOTER //\n// ./src/router/index.js","\r\n
{{errorDialog.text}}\r\n
{{convertedText}}\r\n\r\n\r\n\r\n\n\n\n// WEBPACK FOOTER //\n// src/view/converter.vue","var render = function () {var _vm=this;var _h=_vm.$createElement;var _c=_vm._self._c||_h;return _c('pre',[_vm._v(_vm._s(_vm.convertedText))])}\nvar staticRenderFns = []\nvar esExports = { render: render, staticRenderFns: staticRenderFns }\nexport default esExports\n\n\n//////////////////\n// WEBPACK FOOTER\n// ./node_modules/vue-loader/lib/template-compiler?{\"id\":\"data-v-511afa56\",\"hasScoped\":false,\"transformToRequire\":{\"video\":[\"src\",\"poster\"],\"source\":\"src\",\"img\":\"src\",\"image\":\"xlink:href\"},\"buble\":{\"transforms\":{}}}!./node_modules/vue-loader/lib/selector.js?type=template&index=0!./src/view/converter.vue\n// module id = null\n// module chunks = ","var normalizeComponent = require(\"!../../node_modules/vue-loader/lib/component-normalizer\")\n/* script */\nexport * from \"!!babel-loader!../../node_modules/vue-loader/lib/selector?type=script&index=0!./converter.vue\"\nimport __vue_script__ from \"!!babel-loader!../../node_modules/vue-loader/lib/selector?type=script&index=0!./converter.vue\"\n/* template */\nimport __vue_template__ from \"!!../../node_modules/vue-loader/lib/template-compiler/index?{\\\"id\\\":\\\"data-v-511afa56\\\",\\\"hasScoped\\\":false,\\\"transformToRequire\\\":{\\\"video\\\":[\\\"src\\\",\\\"poster\\\"],\\\"source\\\":\\\"src\\\",\\\"img\\\":\\\"src\\\",\\\"image\\\":\\\"xlink:href\\\"},\\\"buble\\\":{\\\"transforms\\\":{}}}!../../node_modules/vue-loader/lib/selector?type=template&index=0!./converter.vue\"\n/* template functional */\nvar __vue_template_functional__ = false\n/* styles */\nvar __vue_styles__ = null\n/* scopeId */\nvar __vue_scopeId__ = null\n/* moduleIdentifier (server only) */\nvar __vue_module_identifier__ = null\nvar Component = normalizeComponent(\n __vue_script__,\n __vue_template__,\n __vue_template_functional__,\n __vue_styles__,\n __vue_scopeId__,\n __vue_module_identifier__\n)\n\nexport default Component.exports\n\n\n\n//////////////////\n// WEBPACK FOOTER\n// ./src/view/converter.vue\n// module id = null\n// module chunks = ","import Vue from 'vue'\nimport Router from 'vue-router'\nimport home from '@/view/homeView'\nimport converter from '@/view/converter'\n\nVue.use(Router)\n\nexport default new Router({\n routes: [\n {\n path: '/',\n name: 'home',\n component: home\n },\n {\n path: '/converter',\n name: 'converter',\n component: converter\n }\n ]\n})\n\n\n\n// WEBPACK FOOTER //\n// ./src/router/index.js","\r\n
{{errorDialog.text}}\r\n
{{errorDialog.text}}\r\n
{{convertedText}}\r\n\r\n\r\n\r\n\n\n\n// WEBPACK FOOTER //\n// src/view/converter.vue","var render = function () {var _vm=this;var _h=_vm.$createElement;var _c=_vm._self._c||_h;return _c('pre',[_vm._v(_vm._s(_vm.convertedText))])}\nvar staticRenderFns = []\nvar esExports = { render: render, staticRenderFns: staticRenderFns }\nexport default esExports\n\n\n//////////////////\n// WEBPACK FOOTER\n// ./node_modules/vue-loader/lib/template-compiler?{\"id\":\"data-v-511afa56\",\"hasScoped\":false,\"transformToRequire\":{\"video\":[\"src\",\"poster\"],\"source\":\"src\",\"img\":\"src\",\"image\":\"xlink:href\"},\"buble\":{\"transforms\":{}}}!./node_modules/vue-loader/lib/selector.js?type=template&index=0!./src/view/converter.vue\n// module id = null\n// module chunks = ","var normalizeComponent = require(\"!../../node_modules/vue-loader/lib/component-normalizer\")\n/* script */\nexport * from \"!!babel-loader!../../node_modules/vue-loader/lib/selector?type=script&index=0!./converter.vue\"\nimport __vue_script__ from \"!!babel-loader!../../node_modules/vue-loader/lib/selector?type=script&index=0!./converter.vue\"\n/* template */\nimport __vue_template__ from \"!!../../node_modules/vue-loader/lib/template-compiler/index?{\\\"id\\\":\\\"data-v-511afa56\\\",\\\"hasScoped\\\":false,\\\"transformToRequire\\\":{\\\"video\\\":[\\\"src\\\",\\\"poster\\\"],\\\"source\\\":\\\"src\\\",\\\"img\\\":\\\"src\\\",\\\"image\\\":\\\"xlink:href\\\"},\\\"buble\\\":{\\\"transforms\\\":{}}}!../../node_modules/vue-loader/lib/selector?type=template&index=0!./converter.vue\"\n/* template functional */\nvar __vue_template_functional__ = false\n/* styles */\nvar __vue_styles__ = null\n/* scopeId */\nvar __vue_scopeId__ = null\n/* moduleIdentifier (server only) */\nvar __vue_module_identifier__ = null\nvar Component = normalizeComponent(\n __vue_script__,\n __vue_template__,\n __vue_template_functional__,\n __vue_styles__,\n __vue_scopeId__,\n __vue_module_identifier__\n)\n\nexport default Component.exports\n\n\n\n//////////////////\n// WEBPACK FOOTER\n// ./src/view/converter.vue\n// module id = null\n// module chunks = ","import Vue from 'vue'\nimport Router from 'vue-router'\nimport home from '@/view/homeView'\nimport converter from '@/view/converter'\n\nVue.use(Router)\n\nexport default new Router({\n routes: [\n {\n path: '/',\n name: 'home',\n component: home\n },\n {\n path: '/converter',\n name: 'converter',\n component: converter\n }\n ]\n})\n\n\n\n// WEBPACK FOOTER //\n// ./src/router/index.js","import Vue from 'vue'\nimport App from './App'\nimport router from './router'\nimport Vuetify from 'vuetify'\nimport VueResource from 'vue-resource'\n// import chartConstructor from './constructor/2d_graph'\nimport chart2dConstructor from 'vue-chart2d-constructor'\nimport 'vuetify/dist/vuetify.min.css'\n\nVue.use(Vuetify)\n\nVue.config.productionTip = false\nVue.use(VueResource)\nVue.http.headers.common['content-type'] = 'application/json'\nVue.use(chart2dConstructor)\n/* eslint-disable no-new */\nnew Vue({\n el: '#app',\n router,\n render: h => h(App)\n})\n\n\n\n// WEBPACK FOOTER //\n// ./src/main.js","\r\n
{{errorDialog.text}}+