Merge pull request #17 from BBVA/develop

New Release 1.2
BBVA · Nov 20, 2018 · 9ffed30 · 9ffed30
2 parents 32b10f0 + 0dad863
commit 9ffed30
Show file tree

Hide file tree

Showing 102 changed files with 18,662 additions and 2,802 deletions.
diff --git a/README.md b/README.md
@@ -4,7 +4,7 @@ TIMECOP
 ![alt text](https://raw.githubusercontent.com/BBVA/timecop/master/doc/img/timecop.png)
 
 
-TIMECOP is a RESTful webservice engine that evaluates univariate and multivariate timeseries. It considerates that the time series has 3 stages: the current state of the time series as the last five points, the past state as all the previous points before the current state and the future state as the forecast of the next 5 points. 
+TIMECOP is a RESTful webservice engine that evaluates univariate and multivariate timeseries. It considerates that the time series has 3 stages: the current state of the time series as the last five points, the past state as all the previous points before the current state and the future state as the forecast of the next steps(custom number). 
 
 The aim of TIMECOP is to get insight on the behavior of the time series. To achieve this, the engine compares several time series forecasting algorithms and select the best one according to the MAE (mean absolute error) metric. The 3 different algorithms that compounds the engine are: Holt-Winters, ARIMA, and Recurrent Neural Networks using LSTM cells.
 

diff --git a/engines/__init__.pyc b/engines/__init__.pyc
diff --git a/engines/__pycache__/__init__.cpython-36.pyc b/engines/__pycache__/__init__.cpython-36.pyc
diff --git a/engines/__pycache__/auto_arima.cpython-36.pyc b/engines/__pycache__/auto_arima.cpython-36.pyc
diff --git a/engines/__pycache__/functions_timeseries.cpython-36.pyc b/engines/__pycache__/functions_timeseries.cpython-36.pyc
diff --git a/engines/__pycache__/helpers.cpython-36.pyc b/engines/__pycache__/helpers.cpython-36.pyc
diff --git a/engines/__pycache__/holtwinter.cpython-36.pyc b/engines/__pycache__/holtwinter.cpython-36.pyc
diff --git a/engines/__pycache__/lstm.cpython-36.pyc b/engines/__pycache__/lstm.cpython-36.pyc
diff --git a/engines/__pycache__/var.cpython-36.pyc b/engines/__pycache__/var.cpython-36.pyc
diff --git a/engines/auto_arima.py b/engines/auto_arima.py
@@ -2,15 +2,23 @@
 import pandas as pd
 from pyramid.arima import auto_arima
 from sklearn.metrics import mean_squared_error,mean_absolute_error
-import helpers as h
+from . helpers import create_train_test
 
-def anomaly_AutoArima(lista_datos,desv_mse=0):
+def anomaly_AutoArima(lista_datos,num_fut,orig_size,desv_mse=0):
 
-    lista_puntos = np.arange(0, len(lista_datos),1)
+    print (orig_size)
+
+    print ("tamanio original")
+    start_point= int(orig_size) - 100
+
+    print ("start point " + str(start_point))
+    lista_puntos = np.arange(start_point, orig_size,1)
 
-    df, df_train, df_test = h.create_train_test(lista_puntos, lista_datos) 
+    print (lista_puntos)
+    df, df_train, df_test = create_train_test(lista_puntos, lista_datos) 
 
     engine_output={}
+    print ("arranca")
 
     stepwise_model = auto_arima(df_train['valores'], start_p=1, start_q=1, max_p=3, max_q=3, m=12,
                               start_P=0, seasonal=True, d=1, D=1, trace=False, approx=False,
@@ -73,7 +81,7 @@ def anomaly_AutoArima(lista_datos,desv_mse=0):
     ############## FORECAST START
     updated_model = stepwise_model.fit(df['valores'])
 
-    forecast = updated_model.predict(n_periods=5)
+    forecast = updated_model.predict(n_periods=num_fut)
 
     engine_output['rmse'] = rmse
     engine_output['mse'] = mse
@@ -84,11 +92,10 @@ def anomaly_AutoArima(lista_datos,desv_mse=0):
     engine_output['engine']='Autoarima'
     df_future= pd.DataFrame(forecast,columns=['value'])
     df_future['value']=df_future.value.astype("float32")
-    df_future['step']= np.arange( len(lista_datos),len(lista_datos)+5,1)
+    df_future['step']= np.arange( len(lista_datos),len(lista_datos)+num_fut,1)
     engine_output['future'] = df_future.to_dict(orient='record')
     testing_data  = pd.DataFrame(future_forecast_pred,index = df_test.index,columns=['expected value'])
     testing_data['step']=testing_data.index
     engine_output['debug'] = testing_data.to_dict(orient='record')
 
     return (engine_output)
-
diff --git a/engines/auto_arima.pyc b/engines/auto_arima.pyc
diff --git a/engines/functions_timeseries.py b/engines/functions_timeseries.py
@@ -1,55 +1,98 @@
-from helpers import merge_two_dicts
-from var import anomaly_VAR, univariate_anomaly_VAR
-from holtwinter import anomaly_holt
-from auto_arima import anomaly_AutoArima
-from lstm import anomaly_LSTM, anomaly_uni_LSTM
+from engines.helpers import merge_two_dicts
+from . var import anomaly_VAR, univariate_anomaly_VAR,univariate_forecast_VAR
+from . holtwinter import anomaly_holt,forecast_holt
+from . auto_arima import anomaly_AutoArima
+from . lstm import anomaly_LSTM, anomaly_uni_LSTM
+import traceback
 
-def model_univariate(lista_datos,num_fut,desv_mse):
+
+def model_univariate(lista_datos,num_fut,desv_mse,train,name):
     engines_output={}
     debug = {}
 
-    try:
-        engines_output['LSTM'] = anomaly_uni_LSTM(lista_datos,desv_mse)
-        debug['LSTM'] = engines_output['LSTM']['debug']
-    except Exception as e: 
-        print(e)
-        print ('ERROR: exception executing LSTM univariate')
 
-    #try:
-        #engines_output['arima'] = anomaly_AutoArima(lista_datos,desv_mse)
-        #debug['arima'] = engines_output['arima']['debug']
-    #except  Exception as e: 
-        #print(e)
-        #print ('ERROR: exception executing Autoarima')
-    try:
-        engines_output['VAR'] = univariate_anomaly_VAR(lista_datos)
-        debug['VAR'] = engines_output['VAR']['debug']
-    except  Exception as e: 
-        print(e)
-        print ('ERROR: exception executing VAR')
-    try:
-        engines_output['Holtwinters'] = anomaly_holt(lista_datos,desv_mse)
-        debug['Holtwinters'] = engines_output['Holtwinters']['debug']
-    except  Exception as e: 
-        print(e)
-        print ('ERROR: exception executing Holtwinters')
+    if train != 'True':
+        filename = './models_temp/'+name
+        with open(filename,'r') as f:
+            winner = f.read()
+            f.close()
+        if winner == 'LSTM':
+            try:
+                engines_output['LSTM'] = anomaly_uni_LSTM(lista_datos,num_fut,desv_mse,train)
+                debug['LSTM'] = engines_output['LSTM']['debug']
+            except Exception as e: 
+                print(e)
+                print ('ERROR: exception executing LSTM univariate')
+        elif winner == 'VAR':
+            engines_output['VAR'] = univariate_forecast_VAR(lista_datos,num_fut,name)
+            debug['VAR'] = engines_output['VAR']['debug']
+        elif winner == 'Holtwinters':
+            engines_output['Holtwinters'] = forecast_holt(lista_datos,num_fut,desv_mse,name)
+            debug['Holtwinters'] = engines_output['Holtwinters']['debug']
+        else:
+            print ("Error")
 
-    best_mae=999999999
-    winner='Holtwinters'
-    print ('The size is: ')
-    print (len(engines_output))
-    for key, value in engines_output.iteritems():
-        print (key + "   " + str(value['mae']))
+    else:
+        try:
+            engines_output['LSTM'] = anomaly_uni_LSTM(lista_datos,num_fut,desv_mse,train)
+            debug['LSTM'] = engines_output['LSTM']['debug']
+        except Exception as e: 
+            print(e)
+            print ('ERROR: exception executing LSTM univariate')
 
-        if value['mae'] < best_mae:
-            best_mae=value['mae']
-            winner=key
-        print(winner)
-
+        #try:
+            #if (len(lista_datos) > 100):
+                ##new_length= 
+                #lista_datos_ari=lista_datos[len(lista_datos)-100:]
+            #engines_output['arima'] = anomaly_AutoArima(lista_datos_ari,num_fut,len(lista_datos),desv_mse)
+            #debug['arima'] = engines_output['arima']['debug']
+        #except  Exception as e: 
+            #print(e)
+            #print ('ERROR: exception executing Autoarima')
+
+        try:
+            if (train == 'True'):
+                engines_output['VAR'] = univariate_anomaly_VAR(lista_datos,num_fut,name)
+                debug['VAR'] = engines_output['VAR']['debug']
+            else:
+                engines_output['VAR'] = univariate_forecast_VAR(lista_datos,num_fut,name)
+                debug['VAR'] = engines_output['VAR']['debug']
+        except  Exception as e: 
+            print(e)
+            print ('ERROR: exception executing VAR')
+
+        try:
+            if (train == 'True'):
+                engines_output['Holtwinters'] = anomaly_holt(lista_datos,num_fut,desv_mse,name)
+                debug['Holtwinters'] = engines_output['Holtwinters']['debug']
+            else:
+                print ("entra en forecast")
+                engines_output['Holtwinters'] = forecast_holt(lista_datos,num_fut,desv_mse,name)
+                debug['Holtwinters'] = engines_output['Holtwinters']['debug']
+        except  Exception as e: 
+            print(e)
+            print ('ERROR: exception executing Holtwinters')
+
+
+        best_mae=999999999
+        winner='Holtwinters'
+        print ('The size is: ')
+        print (len(engines_output))
+        for key, value in engines_output.items():
+            print (key + "   " + str(value['mae']))
 
+            if value['mae'] < best_mae:
+                best_mae=value['mae']
+                winner=key
+            print(winner)
+
+        filename = './models_temp/'+name
+        with open(filename,'w') as f:
+            f.write(winner)
+            f.close()
 
 
-    print winner
+        print (winner)
 
     temp= {}
     temp['debug']=debug
@@ -64,33 +107,37 @@ def model_multivariate(list_var,num_fut,desv_mse):
     debug = {}
 
     try:
-        engines_output['LSTM'] = anomaly_LSTM(list_var,desv_mse)
+        engines_output['LSTM'] = anomaly_LSTM(list_var,num_fut,desv_mse)
         debug['LSTM'] = engines_output['LSTM']['debug']
         print (engines_output['LSTM'])
     except   Exception as e: 
         print(e)
         print ('ERROR: exception executing LSTM')
+
     try:
-        engines_output['VAR'] = anomaly_VAR(list_var)
+        engines_output['VAR'] = anomaly_VAR(list_var,num_fut)
         debug['VAR'] = engines_output['VAR']['debug']
         print (engines_output['VAR'])
     except   Exception as e: 
-        print(e)
-        print ('ERROR: exception executing VAR')    
+        print(Exception)
+        print("type error: " + str(e))
+        print(traceback.format_exc())
+        print ('ERROR: exception executing VAR') 
+
     best_mae=999999999
     winner='LSTM'
     print ('The size is ')
     print (len(engines_output))
     print (debug)
-    for key, value in engines_output.iteritems():
+    for key, value in engines_output.items():
         print (key)
         print(str(value['mae']))
         if value['mae'] < best_mae:
             print (key + " " + str(value['mae']) + " best:" + str(best_mae) )
             best_mae=value['mae']
             winner=key
 
-    print "el ganador es " + winner
+    print ("el ganador es " + winner)
     temp= {}
     temp['debug']=debug
     return merge_two_dicts(engines_output[winner] , temp)
diff --git a/engines/functions_timeseries.pyc b/engines/functions_timeseries.pyc
diff --git a/engines/helpers.pyc b/engines/helpers.pyc