Merge pull request #148 from antoinecarme/missing_data

Missing data

pyaf/TS/MissingData.py

@@ -55,6 +55,27 @@ def apply_signal_imputation_method(self, iInputDS, iSignal):
         elif(self.mOptions.mMissingDataOptions.mSignalMissingDataImputation == "Interpolate"):
             lSignal = self.interpolate_signal_if_needed(iInputDS , iSignal)
             iInputDS[iSignal] = lSignal
+
+        elif(self.mOptions.mMissingDataOptions.mSignalMissingDataImputation == "Constant"):
+            lSignal = iInputDS[iSignal].fillna(self.mOptions.mMissingDataOptions.mConstant, method=None)
+            iInputDS[iSignal] = lSignal
+
+        elif(self.mOptions.mMissingDataOptions.mSignalMissingDataImputation == "Mean"):
+            lMean = iInputDS[iSignal].mean()
+            lSignal = iInputDS[iSignal].fillna(lMean, method=None)
+            iInputDS[iSignal] = lSignal
+
+        elif(self.mOptions.mMissingDataOptions.mSignalMissingDataImputation == "Median"):
+            lMedian = iInputDS[iSignal].median()
+            lSignal = iInputDS[iSignal].fillna(lMedian, method=None)
+            iInputDS[iSignal] = lSignal
+
+        elif(self.mOptions.mMissingDataOptions.mSignalMissingDataImputation == "PreviousValue"):
+            lSignal = iInputDS[iSignal].fillna(method='ffill')
+            # replace the first empty values with the first known value
+            lSignal = lSignal.fillna(method='bfill')
+            iInputDS[iSignal] = lSignal
+
         return iInputDS
 
     def interpolate_time_if_needed(self, iInputDS , iTime):

pyaf/TS/Options.py

@@ -126,8 +126,9 @@ def __init__(self):
 class cMissingDataOptions:
 
     def __init__(self):
-        self.mSignalMissingDataImputation = None  # [None , "DiscardRow", "Interpolate"]
+        self.mSignalMissingDataImputation = None  # [None , "DiscardRow", "Interpolate", "Mean", "Median" , "Constant" , "PreviousValue"]
         self.mTimeMissingDataImputation = None  # [None , "DiscardRow", "Interpolate"]
+        self.mConstant = 0.0
 
 class cSignalDecomposition_Options(cModelControl):
 

pyaf/TS/SignalDecomposition.py

@@ -745,7 +745,10 @@ def standardPlots(self, name = None, format = 'png'):
         start_time = time.time()
         logger.info("START_PLOTTING")
         for lSignal in self.mSignals:
-            self.mBestModels[lSignal].standardPlots(name + "_" + lSignal, format);
+            lName = name
+            if(name is not None):
+                lName = str(name) + "_" + str(lSignal)
+            self.mBestModels[lSignal].standardPlots(lName, format);
         lPlotTime = time.time() - start_time;
         logger.info("END_PLOTTING_TIME_IN_SECONDS " + str(lPlotTime))
 

tests/bugs/issue_55/grouping_issue_55_notebook.py

@@ -114,7 +114,7 @@
 French_Wine_Export_in_Euros_DF.info()
 
 
-lInfo = lEngine.to_json()
+lInfo = lEngine.to_dict()
 print(lInfo.keys())
 
 print(lInfo['Structure'])
@@ -133,7 +133,7 @@
 
 lEngine.mSignalHierarchy.plot()
 
-CN_Engine = lEngine.mSignalHierarchy.mModels[2]['__CN'] # __CN is at hierarchical level 2
+CN_Engine = lEngine.mSignalHierarchy.mModels # __CN is at hierarchical level 2
 
 CN_Engine.getModelInfo()
 

tests/gen_makefile.py

@@ -27,7 +27,7 @@ def add_makefile_entry(subdir1):
     return test_target;
 
 
-str1 = "artificial basic_checks bugs cross_validation croston exog expsmooth func HeartRateTimeSeries heroku hierarchical  HourOfWeek model_control perf svr transformations  neuralnet real-life  time_res perfs demos xgb xeon-phi-parallel sampling temporal_hierarchy WeekOfMonth";
+str1 = "artificial basic_checks bugs cross_validation croston exog expsmooth func HeartRateTimeSeries heroku hierarchical  HourOfWeek model_control perf svr transformations  neuralnet real-life  time_res perfs demos xgb xeon-phi-parallel sampling temporal_hierarchy WeekOfMonth missing_data";
 subdirs = str1.split();
 
 print("PYTHON=python3\n\n");
@@ -45,6 +45,6 @@ def add_makefile_entry(subdir1):
 
 print("all: " , str1 , "\n\t\n");
 
-str2 = "demos basic_checks cross_validation croston exog heroku hierarchical model_control perfs svr transformations func real-life  time_res xgb sampling HourOfWeek WeekOfMonth";
+str2 = "demos basic_checks cross_validation croston exog heroku hierarchical model_control perfs svr transformations func real-life  time_res xgb sampling HourOfWeek WeekOfMonth missing_data";
 
 print("build-test : " , str2 , "\n\t\n");

tests/missing_data/gen_air_passengers_tests.py

@@ -0,0 +1,16 @@
+
+
+
+
+
+def gen_all_air_passengers():
+    lDir = "tests/missing_data"
+    for iTimeImp in [None , "DiscardRow", "Interpolate"]:
+        for iSigImp in [None , "DiscardRow", "Interpolate", "Mean", "Median" , "Constant" , "PreviousValue"]:
+            filename = lDir + "/test_missing_data_air_passengers_" + str(iTimeImp) + "_" + str(iSigImp) + ".py";
+            with open(filename, "w") as outfile:
+                print("WRTITING_FILE" , filename)
+                outfile.write("import tests.missing_data.test_missing_data_air_passengers_generic as gen\n\n")
+                outfile.write("gen.test_air_passengers_missing_data" + str((iTimeImp , iSigImp)) + "\n")
+
+# gen_all_air_passengers()

tests/missing_data/gen_ozone_tests.py

@@ -0,0 +1,16 @@
+
+
+
+
+
+def gen_all_ozone():
+    lDir = "tests/missing_data"
+    for iTimeImp in [None , "DiscardRow", "Interpolate"]:
+        for iSigImp in [None , "DiscardRow", "Interpolate", "Mean", "Median" , "Constant" , "PreviousValue"]:
+            filename = lDir + "/test_missing_data_ozone_" + str(iTimeImp) + "_" + str(iSigImp) + ".py";
+            with open(filename, "w") as outfile:
+                print("WRTITING_FILE" , filename)
+                outfile.write("import tests.missing_data.test_missing_data_ozone_generic as gen\n\n")
+                outfile.write("gen.test_ozone_missing_data" + str((iTimeImp , iSigImp)) + "\n")
+
+# gen_all_ozone()

tests/missing_data/test_missing_data_air_passengers_DiscardRow_Constant.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_air_passengers_generic as gen
+
+gen.test_air_passengers_missing_data('DiscardRow', 'Constant')

tests/missing_data/test_missing_data_air_passengers_DiscardRow_DiscardRow.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_air_passengers_generic as gen
+
+gen.test_air_passengers_missing_data('DiscardRow', 'DiscardRow')

tests/missing_data/test_missing_data_air_passengers_DiscardRow_Interpolate.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_air_passengers_generic as gen
+
+gen.test_air_passengers_missing_data('DiscardRow', 'Interpolate')

tests/missing_data/test_missing_data_air_passengers_DiscardRow_Mean.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_air_passengers_generic as gen
+
+gen.test_air_passengers_missing_data('DiscardRow', 'Mean')

tests/missing_data/test_missing_data_air_passengers_DiscardRow_Median.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_air_passengers_generic as gen
+
+gen.test_air_passengers_missing_data('DiscardRow', 'Median')

tests/missing_data/test_missing_data_air_passengers_DiscardRow_None.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_air_passengers_generic as gen
+
+gen.test_air_passengers_missing_data('DiscardRow', None)

tests/missing_data/test_missing_data_air_passengers_DiscardRow_PreviousValue.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_air_passengers_generic as gen
+
+gen.test_air_passengers_missing_data('DiscardRow', 'PreviousValue')

tests/missing_data/test_missing_data_air_passengers_Interpolate_Constant.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_air_passengers_generic as gen
+
+gen.test_air_passengers_missing_data('Interpolate', 'Constant')

tests/missing_data/test_missing_data_air_passengers_Interpolate_DiscardRow.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_air_passengers_generic as gen
+
+gen.test_air_passengers_missing_data('Interpolate', 'DiscardRow')

tests/missing_data/test_missing_data_air_passengers_Interpolate_Interpolate.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_air_passengers_generic as gen
+
+gen.test_air_passengers_missing_data('Interpolate', 'Interpolate')

tests/missing_data/test_missing_data_air_passengers_Interpolate_Mean.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_air_passengers_generic as gen
+
+gen.test_air_passengers_missing_data('Interpolate', 'Mean')

tests/missing_data/test_missing_data_air_passengers_Interpolate_Median.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_air_passengers_generic as gen
+
+gen.test_air_passengers_missing_data('Interpolate', 'Median')

tests/missing_data/test_missing_data_air_passengers_Interpolate_None.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_air_passengers_generic as gen
+
+gen.test_air_passengers_missing_data('Interpolate', None)

tests/missing_data/test_missing_data_air_passengers_Interpolate_PreviousValue.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_air_passengers_generic as gen
+
+gen.test_air_passengers_missing_data('Interpolate', 'PreviousValue')

tests/missing_data/test_missing_data_air_passengers_None_Constant.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_air_passengers_generic as gen
+
+gen.test_air_passengers_missing_data(None, 'Constant')

tests/missing_data/test_missing_data_air_passengers_None_DiscardRow.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_air_passengers_generic as gen
+
+gen.test_air_passengers_missing_data(None, 'DiscardRow')

tests/missing_data/test_missing_data_air_passengers_None_Interpolate.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_air_passengers_generic as gen
+
+gen.test_air_passengers_missing_data(None, 'Interpolate')

tests/missing_data/test_missing_data_air_passengers_None_Mean.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_air_passengers_generic as gen
+
+gen.test_air_passengers_missing_data(None, 'Mean')

tests/missing_data/test_missing_data_air_passengers_None_Median.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_air_passengers_generic as gen
+
+gen.test_air_passengers_missing_data(None, 'Median')

tests/missing_data/test_missing_data_air_passengers_None_None.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_air_passengers_generic as gen
+
+gen.test_air_passengers_missing_data(None, None)

tests/missing_data/test_missing_data_air_passengers_None_PreviousValue.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_air_passengers_generic as gen
+
+gen.test_air_passengers_missing_data(None, 'PreviousValue')

tests/missing_data/test_missing_data_air_passengers_generic.py

@@ -0,0 +1,54 @@
+
+import pandas as pd
+import numpy as np
+
+
+import pyaf.ForecastEngine as autof
+import pyaf.Bench.TS_datasets as tsds
+
+def add_some_missing_data_in_signal(df, col):
+    lRate = 0.2
+    df.loc[df.sample(frac=lRate, random_state=1960).index, col] = np.nan
+    return df
+
+def add_some_missing_data_in_time(df, col):
+    lRate = 0.2
+    df.loc[df.sample(frac=lRate, random_state=1960).index, col] = np.nan
+    return df
+
+
+def test_air_passengers_missing_data(iTimeMissingDataImputation, iSignalMissingDataImputation):
+
+    b1 = tsds.load_airline_passengers()
+    df = b1.mPastData
+
+    if(iTimeMissingDataImputation is not None):
+        df = add_some_missing_data_in_time(df, b1.mTimeVar)
+    if(iSignalMissingDataImputation is not None):
+        df = add_some_missing_data_in_signal(df, b1.mSignalVar)
+
+    lEngine = autof.cForecastEngine()
+    H = b1.mHorizon;
+    lEngine.mOptions.mMissingDataOptions.mTimeMissingDataImputation = iTimeMissingDataImputation
+    lEngine.mOptions.mMissingDataOptions.mSignalMissingDataImputation = iSignalMissingDataImputation
+    lEngine.train(df , b1.mTimeVar , b1.mSignalVar, H);
+    lEngine.getModelInfo();
+    print(lEngine.mSignalDecomposition.mTrPerfDetails.head());
+
+    dfapp_in = df.copy();
+    dfapp_in.tail()
+    dfapp_out = lEngine.forecast(dfapp_in, H);
+    #dfapp_out.to_csv("outputs/ozone_apply_out.csv")
+    dfapp_out.tail(2 * H)
+    print("Forecast Columns " , dfapp_out.columns);
+    Forecast_DF = dfapp_out[[b1.mTimeVar , b1.mSignalVar, b1.mSignalVar + '_Forecast']]
+    print(Forecast_DF.info())
+    print("Forecasts\n" , Forecast_DF.tail(H));
+
+    print("\n\n<ModelInfo>")
+    print(lEngine.to_json());
+    print("</ModelInfo>\n\n")
+    print("\n\n<Forecast>")
+    print(Forecast_DF.tail(2*H).to_json(date_format='iso'))
+    print("</Forecast>\n\n")
+

tests/missing_data/test_missing_data_ozone_DiscardRow_Constant.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_ozone_generic as gen
+
+gen.test_ozone_missing_data('DiscardRow', 'Constant')

tests/missing_data/test_missing_data_ozone_DiscardRow_DiscardRow.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_ozone_generic as gen
+
+gen.test_ozone_missing_data('DiscardRow', 'DiscardRow')

tests/missing_data/test_missing_data_ozone_DiscardRow_Interpolate.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_ozone_generic as gen
+
+gen.test_ozone_missing_data('DiscardRow', 'Interpolate')

tests/missing_data/test_missing_data_ozone_DiscardRow_Mean.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_ozone_generic as gen
+
+gen.test_ozone_missing_data('DiscardRow', 'Mean')

tests/missing_data/test_missing_data_ozone_DiscardRow_Median.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_ozone_generic as gen
+
+gen.test_ozone_missing_data('DiscardRow', 'Median')

tests/missing_data/test_missing_data_ozone_DiscardRow_None.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_ozone_generic as gen
+
+gen.test_ozone_missing_data('DiscardRow', None)

tests/missing_data/test_missing_data_ozone_DiscardRow_PreviousValue.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_ozone_generic as gen
+
+gen.test_ozone_missing_data('DiscardRow', 'PreviousValue')

tests/missing_data/test_missing_data_ozone_Interpolate_Constant.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_ozone_generic as gen
+
+gen.test_ozone_missing_data('Interpolate', 'Constant')

tests/missing_data/test_missing_data_ozone_Interpolate_DiscardRow.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_ozone_generic as gen
+
+gen.test_ozone_missing_data('Interpolate', 'DiscardRow')

tests/missing_data/test_missing_data_ozone_Interpolate_Interpolate.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_ozone_generic as gen
+
+gen.test_ozone_missing_data('Interpolate', 'Interpolate')

tests/missing_data/test_missing_data_ozone_Interpolate_Mean.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_ozone_generic as gen
+
+gen.test_ozone_missing_data('Interpolate', 'Mean')

tests/missing_data/test_missing_data_ozone_Interpolate_Median.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_ozone_generic as gen
+
+gen.test_ozone_missing_data('Interpolate', 'Median')

tests/missing_data/test_missing_data_ozone_Interpolate_None.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_ozone_generic as gen
+
+gen.test_ozone_missing_data('Interpolate', None)

tests/missing_data/test_missing_data_ozone_Interpolate_PreviousValue.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_ozone_generic as gen
+
+gen.test_ozone_missing_data('Interpolate', 'PreviousValue')

tests/missing_data/test_missing_data_ozone_None_Constant.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_ozone_generic as gen
+
+gen.test_ozone_missing_data(None, 'Constant')

tests/missing_data/test_missing_data_ozone_None_DiscardRow.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_ozone_generic as gen
+
+gen.test_ozone_missing_data(None, 'DiscardRow')

tests/missing_data/test_missing_data_ozone_None_Interpolate.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_ozone_generic as gen
+
+gen.test_ozone_missing_data(None, 'Interpolate')

tests/missing_data/test_missing_data_ozone_None_Mean.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_ozone_generic as gen
+
+gen.test_ozone_missing_data(None, 'Mean')

tests/missing_data/test_missing_data_ozone_None_Median.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_ozone_generic as gen
+
+gen.test_ozone_missing_data(None, 'Median')

tests/missing_data/test_missing_data_ozone_None_None.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_ozone_generic as gen
+
+gen.test_ozone_missing_data(None, None)

tests/missing_data/test_missing_data_ozone_None_PreviousValue.py

@@ -0,0 +1,3 @@
+import tests.missing_data.test_missing_data_ozone_generic as gen
+
+gen.test_ozone_missing_data(None, 'PreviousValue')

tests/missing_data/test_missing_data_ozone_generic.py

@@ -0,0 +1,54 @@
+
+import pandas as pd
+import numpy as np
+
+
+import pyaf.ForecastEngine as autof
+import pyaf.Bench.TS_datasets as tsds
+
+def add_some_missing_data_in_signal(df, col):
+    lRate = 0.2
+    df.loc[df.sample(frac=lRate, random_state=1960).index, col] = np.nan
+    return df
+
+def add_some_missing_data_in_time(df, col):
+    lRate = 0.2
+    df.loc[df.sample(frac=lRate, random_state=1960).index, col] = np.nan
+    return df
+
+
+def test_ozone_missing_data(iTimeMissingDataImputation, iSignalMissingDataImputation):
+
+    b1 = tsds.load_ozone()
+    df = b1.mPastData
+
+    if(iTimeMissingDataImputation is not None):
+        df = add_some_missing_data_in_time(df, b1.mTimeVar)
+    if(iSignalMissingDataImputation is not None):
+        df = add_some_missing_data_in_signal(df, b1.mSignalVar)
+
+    lEngine = autof.cForecastEngine()
+    H = b1.mHorizon;
+    lEngine.mOptions.mMissingDataOptions.mTimeMissingDataImputation = iTimeMissingDataImputation
+    lEngine.mOptions.mMissingDataOptions.mSignalMissingDataImputation = iSignalMissingDataImputation
+    lEngine.train(df , b1.mTimeVar , b1.mSignalVar, H);
+    lEngine.getModelInfo();
+    print(lEngine.mSignalDecomposition.mTrPerfDetails.head());
+
+    dfapp_in = df.copy();
+    dfapp_in.tail()
+    dfapp_out = lEngine.forecast(dfapp_in, H);
+    #dfapp_out.to_csv("outputs/ozone_apply_out.csv")
+    dfapp_out.tail(2 * H)
+    print("Forecast Columns " , dfapp_out.columns);
+    Forecast_DF = dfapp_out[[b1.mTimeVar , b1.mSignalVar, b1.mSignalVar + '_Forecast']]
+    print(Forecast_DF.info())
+    print("Forecasts\n" , Forecast_DF.tail(H));
+
+    print("\n\n<ModelInfo>")
+    print(lEngine.to_json());
+    print("</ModelInfo>\n\n")
+    print("\n\n<Forecast>")
+    print(Forecast_DF.tail(2*H).to_json(date_format='iso'))
+    print("</Forecast>\n\n")
+