Preprocess/ndvi

.vscode/launch.json

@@ -0,0 +1,15 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python: Current File",
+            "type": "python",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal"
+        }
+    ]
+}

.vscode/settings.json

@@ -0,0 +1,7 @@
+{
+    "python.pythonPath": "/Users/tommylees/miniconda3/envs/crop/bin/python",
+    "python.testing.pytestArgs": ["tests"],
+    "python.testing.unittestEnabled": false,
+    "python.testing.nosetestsEnabled": false,
+    "python.testing.pytestEnabled": true
+}

notebooks/draft/05_tl_event_detect.ipynb

@@ -3619,7 +3619,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.2"
+   "version": "3.7.0"
   }
  },
  "nbformat": 4,

notebooks/draft/06_tl_indices.ipynb

@@ -43,7 +43,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.2"
+   "version": "3.7.0"
   }
  },
  "nbformat": 4,

scripts/drafts/eng_utils.py

@@ -15,6 +15,8 @@
 import pandas as pd
 from netCDF4 import num2date
 
+from pathlib import Path
+from typing import List
 
 # ------------------------------------------------------------------------------
 # Selcting the Same Timeslice

scripts/engineer.py

@@ -5,8 +5,8 @@
 from src.engineer import Engineer
 
 
-def engineer(experiment='one_month_forecast', process_static=True,
-             pred_months=12):
+def engineer_VCI(experiment='one_month_forecast', process_static=True,
+                 pred_months=12):
     # if the working directory is alread ml_drought don't need ../data
     if Path('.').absolute().as_posix().split('/')[-1] == 'ml_drought':
         data_path = Path('data')
@@ -15,7 +15,23 @@ def engineer(experiment='one_month_forecast', process_static=True,
 
     engineer = Engineer(data_path, experiment=experiment, process_static=process_static)
     engineer.engineer(
-        test_year=2018, target_variable='VCI',
+        test_year=[y for y in range(2011, 2019)], target_variable='VCI',
+        pred_months=pred_months, expected_length=pred_months,
+    )
+
+
+def engineer_NDVI(experiment='one_month_forecast', process_static=True,
+                  pred_months=12):
+    # if the working directory is alread ml_drought don't need ../data
+    if Path('.').absolute().as_posix().split('/')[-1] == 'ml_drought':
+        data_path = Path('data')
+    else:
+        data_path = Path('../data')
+
+    engineer = Engineer(data_path, experiment=experiment,
+                        process_static=process_static)
+    engineer.engineer(
+        test_year=[y for y in range(2011, 2019)], target_variable='ndvi',
         pred_months=pred_months, expected_length=pred_months,
     )
 
@@ -30,5 +46,6 @@ def engineer_static():
 
 
 if __name__ == '__main__':
-    engineer(pred_months=12)
+    # engineer_NDVI(pred_months=12)
+    engineer_VCI(pred_months=3)
     # engineer_static()

scripts/export.py

@@ -43,14 +43,15 @@ def export_era5():
     }
 
     era5_variables = [
-        '10m_u_component_of_wind', '10m_v_component_of_wind', 'volumetric_soil_water_layer_1',
-        'volumetric_soil_water_layer_2', 'volumetric_soil_water_layer_3',
-        'volumetric_soil_water_layer_4', 'surface_pressure', 'surface_sensible_heat_flux',
-        'surface_latent_heat_flux', 'soil_temperature_level_1', '2m_temperature',
-        'mean_eastward_turbulent_surface_stress', 'mean_northward_turbulent_surface_stress',
-        'surface_net_solar_radiation_clear_sky', 'surface_net_thermal_radiation_clear_sky',
-        'vertical_integral_of_divergence_of_moisture_flux', 'potential_evaporation',
-        'evaporation'
+        # '10m_u_component_of_wind', '10m_v_component_of_wind',
+        'volumetric_soil_water_layer_1', 'volumetric_soil_water_layer_2',
+        'volumetric_soil_water_layer_3', 'volumetric_soil_water_layer_4',
+        # 'surface_pressure', 'surface_sensible_heat_flux',
+        # 'surface_latent_heat_flux', 'soil_temperature_level_1', '2m_temperature',
+        # 'mean_eastward_turbulent_surface_stress', 'mean_northward_turbulent_surface_stress',
+        # 'surface_net_solar_radiation_clear_sky', 'surface_net_thermal_radiation_clear_sky',
+        # 'vertical_integral_of_divergence_of_moisture_flux', 'potential_evaporation',
+        # 'evaporation'
     ]
 
     for variable in era5_variables:
@@ -172,10 +173,10 @@ def export_kenya_boundaries():
 
 if __name__ == '__main__':
     export_era5()
-    export_vhi()
-    export_chirps()
-    export_era5POS()
-    export_gleam()
-    export_esa()
-    export_s5()
-    export_kenya_boundaries()
+    # export_vhi()
+    # export_chirps()
+    # export_era5POS()
+    # export_gleam()
+    # export_esa()
+    # export_s5()
+    # export_kenya_boundaries()

scripts/models.py

@@ -1,11 +1,14 @@
 import sys
 sys.path.append('..')
 
-from pathlib import Path
+from src.analysis import all_shap_for_file
 from src.models import (Persistence, LinearRegression,
                         LinearNetwork, RecurrentNetwork,
                         EARecurrentNetwork, load_model)
-from src.analysis import all_shap_for_file
+from pathlib import Path
+
+
+# NOTE: p84.162 == 'vertical integral of moisture flux'
 
 
 def parsimonious(
@@ -17,14 +20,17 @@ def parsimonious(
     else:
         data_path = Path('../data')
 
-    predictor = Persistence(data_path, experiment=experiment)
+    predictor = Persistence(
+        data_path, experiment=experiment
+    )
     predictor.evaluate(save_preds=True)
 
 
 def regression(
     experiment='one_month_forecast',
     include_pred_month=True,
-    surrounding_pixels=1
+    surrounding_pixels=None,
+    ignore_vars=None
 ):
     # if the working directory is alread ml_drought don't need ../data
     if Path('.').absolute().as_posix().split('/')[-1] == 'ml_drought':
@@ -35,19 +41,21 @@ def regression(
     predictor = LinearRegression(
         data_path, experiment=experiment,
         include_pred_month=include_pred_month,
-        surrounding_pixels=surrounding_pixels
+        surrounding_pixels=surrounding_pixels,
+        ignore_vars=ignore_vars,
     )
-    predictor.train()
+    predictor.train(early_stopping=5)
     predictor.evaluate(save_preds=True)
 
     # mostly to test it works
-    predictor.explain(save_shap_values=True)
+    # predictor.explain(save_shap_values=True)
 
 
 def linear_nn(
     experiment='one_month_forecast',
     include_pred_month=True,
-    surrounding_pixels=1
+    surrounding_pixels=None,
+    ignore_vars=None
 ):
     # if the working directory is alread ml_drought don't need ../data
     if Path('.').absolute().as_posix().split('/')[-1] == 'ml_drought':
@@ -59,19 +67,21 @@ def linear_nn(
         layer_sizes=[100], data_folder=data_path,
         experiment=experiment,
         include_pred_month=include_pred_month,
-        surrounding_pixels=surrounding_pixels
+        surrounding_pixels=surrounding_pixels,
+        ignore_vars=ignore_vars,
     )
     predictor.train(num_epochs=50, early_stopping=5)
     predictor.evaluate(save_preds=True)
     predictor.save_model()
 
-    _ = predictor.explain(save_shap_values=True)
+    # _ = predictor.explain(save_shap_values=True)
 
 
 def rnn(
     experiment='one_month_forecast',
     include_pred_month=True,
-    surrounding_pixels=1
+    surrounding_pixels=None,
+    ignore_vars=None
 ):
     # if the working directory is alread ml_drought don't need ../data
     if Path('.').absolute().as_posix().split('/')[-1] == 'ml_drought':
@@ -84,20 +94,22 @@ def rnn(
         data_folder=data_path,
         experiment=experiment,
         include_pred_month=include_pred_month,
-        surrounding_pixels=surrounding_pixels
+        surrounding_pixels=surrounding_pixels,
+        ignore_vars=ignore_vars,
     )
     predictor.train(num_epochs=50, early_stopping=5)
     predictor.evaluate(save_preds=True)
     predictor.save_model()
 
-    _ = predictor.explain(save_shap_values=True)
+    # _ = predictor.explain(save_shap_values=True)
 
 
 def earnn(
     experiment='one_month_forecast',
     include_pred_month=True,
     surrounding_pixels=None,
-    pretrained=True
+    pretrained=True,
+    ignore_vars=None
 ):
     # if the working directory is alread ml_drought don't need ../data
     if Path('.').absolute().as_posix().split('/')[-1] == 'ml_drought':
@@ -111,22 +123,27 @@ def earnn(
             data_folder=data_path,
             experiment=experiment,
             include_pred_month=include_pred_month,
-            surrounding_pixels=surrounding_pixels
+            surrounding_pixels=surrounding_pixels,
+            ignore_vars=ignore_vars,
         )
-        predictor.train(num_epochs=50, early_stopping=5)
+        predictor.train(num_epochs=50, early_stopping=10)
         predictor.evaluate(save_preds=True)
         predictor.save_model()
     else:
-        predictor = load_model(data_path / f'models/{experiment}/ealstm/model.pt')
+        predictor = load_model(
+            data_path / f'models/{experiment}/ealstm/model.pt')
 
-    test_file = data_path / f'features/{experiment}/test/2018_3'
-    assert test_file.exists()
-    all_shap_for_file(test_file, predictor, batch_size=100)
+    # test_file = data_path / f'features/{experiment}/test/2018_3'
+    # assert test_file.exists()
+    # all_shap_for_file(test_file, predictor, batch_size=100)
 
 
 if __name__ == '__main__':
-    # parsimonious()
-    # regression()
-    # linear_nn()
-    # rnn()
-    earnn(pretrained=True)
+    ignore_vars = None
+    ignore_vars = ['ndvi', 'p84.162', 'sp', 'tp']
+
+    parsimonious()
+    regression(ignore_vars=ignore_vars)
+    linear_nn(ignore_vars=ignore_vars)
+    # rnn(ignore_vars=ignore_vars)
+    # earnn(pretrained=False, ignore_vars=ignore_vars)