Made an mlp baseline that actually scores surprisingly well in ndcg on the large test set. It's competitive with our other top models.

dna/models/__init__.py

@@ -2,7 +2,7 @@
 
 from .baselines import (
     AutoSklearnMetalearner, LinearRegressionBaseline, MeanBaseline, MedianBaseline, MetaAutoSklearn,
-    PerPrimitiveBaseline, RandomBaseline, RandomForestBaseline, MLPRegressionModel
+    PerPrimitiveBaseline, RandomBaseline, RandomForestBaseline, MLPBaseline, MLPAblationModel
 )
 from .dna_regression_model import DNARegressionModel
 from .lstm_model import LSTMModel
@@ -31,7 +31,8 @@ def get_model_class(model_id: str):
         'dag_attention_regression': DAGAttentionRegressionModel,
         'linear_regression': LinearRegressionBaseline,
         'random_forest': RandomForestBaseline,
-        'mlp_regression': MLPRegressionModel,
+        'mlp_baseline': MLPBaseline,
+        'mlp_ablation': MLPAblationModel,
         'random': RandomBaseline,
         'meta_autosklearn': MetaAutoSklearn,
         'probabilistic_matrix_factorization': ProbabilisticMatrixFactorization,

dna/models/baselines.py

@@ -6,6 +6,7 @@
 import pandas as pd
 from sklearn import linear_model
 from sklearn.ensemble import RandomForestRegressor
+from sklearn.neural_network import MLPRegressor
 import torch
 from .torch_modules.mlp import MLP
 
@@ -151,6 +152,24 @@ def __init__(self, seed=0):
         self.fitted = False
 
 
+class MLPBaseline(SklearnBase):
+    """
+    Takes in a vector of metafeatures concatenated to binary nominal features that represent which primitives are in
+    a pipeline. This is passed into sklearn's MLPRegressor.
+    See https://scikit-learn.org/stable/modules/generated/sklearn.neural_network.MLPClassifier.html for a list of all
+    the constructor inputs, most of which can be used as tunable hyper-parameters
+    """
+
+    def __init__(self, seed=0, *, hidden_layer_size, n_hidden_layers, **kwargs):
+        super().__init__(seed=seed)
+        hidden_layer_sizes = [hidden_layer_size] * n_hidden_layers
+        self.regressor = MLPRegressor(
+            random_state=seed, early_stopping=True, hidden_layer_sizes=hidden_layer_sizes,
+            **kwargs
+        )
+        self.fitted = False
+
+
 class MetaAutoSklearn(SklearnBase):
 
     # used for plotting and reporting
@@ -249,7 +268,7 @@ def _process_metafeatures(data):
         metafeatures.drop_duplicates(inplace=True)
         return metafeatures
 
-class MLPRegressionModel(PyTorchRegressionRankModelBase):
+class MLPAblationModel(PyTorchRegressionRankModelBase):
 
     def __init__(
             self, n_hidden_layers: int, hidden_layer_size: int, activation_name: str, use_batch_norm: bool,

main.sh

@@ -100,8 +100,8 @@ python3 -m dna evaluate \
 
 # mlp outputs a constant value for a dataset, so it cannot rank
 python3 -m dna evaluate \
-    --model mlp_regression \
-    --model-config-path ./model_configs/mlp_regression_config.json \
+    --model mlp_ablation \
+    --model-config-path ./model_configs/mlp_ablation_config.json \
     --problem regression \
     --metafeature-subset $metafeature_subset \
     --train-path $train_path \
@@ -111,6 +111,18 @@ python3 -m dna evaluate \
     $use_ootsp
 
 
+python3 -m dna evaluate \
+    --model mlp_baseline \
+    --model-config-path ./model_configs/mlp_baseline_config.json \
+    --problem regression rank\
+    --metafeature-subset $metafeature_subset \
+    --train-path $train_path \
+    --test-path $test_path \
+    --output-dir $results_dir \
+    --verbose \
+    $use_ootsp
+
+
 python3 -m dna evaluate \
     --model meta_autosklearn \
     --model-config-path ./model_configs/meta_autosklearn_config.json \

model_configs/mlp_baseline_config.json

@@ -0,0 +1,12 @@
+{
+    "__init__": {
+        "activation": "relu",
+        "hidden_layer_size": 100,
+        "n_hidden_layers": 1,
+
+        "batch_size": 25,
+        "momentum": 0.9,
+        "learning_rate": "constant",
+        "learning_rate_init": 0.001
+    }
+}

test/model_configs/mlp_baseline_config.json

@@ -0,0 +1,12 @@
+{
+    "__init__": {
+        "activation": "relu",
+        "hidden_layer_size": 2,
+        "n_hidden_layers": 1,
+
+        "batch_size": 25,
+        "momentum": 0.9,
+        "learning_rate": "constant",
+        "learning_rate_init": 0.001
+    }
+}

test/test_models.py

@@ -35,14 +35,19 @@ def test_dna_regression_determinism(self):
             model='dna_regression', model_config_path='./test/model_configs/dna_regression_config.json'
         )
 
+    def test_mlp_regression_determinism(self):
+        self._test_determinism(
+            model='mlp_baseline', model_config_path='./test/model_configs/mlp_baseline_config.json', pytorch_model=False
+        )
+
     def test_daglstm_regression_determinism(self):
         # TODO: fix this test on the CPU
         if torch.cuda.is_available():
             self._test_determinism(
                 model='daglstm_regression', model_config_path='./test/model_configs/daglstm_regression_config.json'
             )
 
-    def _test_determinism(self, model: str, model_config_path: str):
+    def _test_determinism(self, model: str, model_config_path: str, pytorch_model: bool = True):
         # Set the arguments for this test
         parser = argparse.ArgumentParser()
         configure_evaluate_parser(parser)
@@ -59,19 +64,19 @@ def _test_determinism(self, model: str, model_config_path: str):
         ]
         arguments = parser.parse_args(argv)
 
-        results1 = self._evaluate_model(arguments)
-        results2 = self._evaluate_model(arguments)
+        results1 = self._evaluate_model(arguments, pytorch_model)
+        results2 = self._evaluate_model(arguments, pytorch_model)
         self.assertEqual(results1, results2)
 
     @staticmethod
-    def _evaluate_model(arguments):
+    def _evaluate_model(arguments, pytorch_model: bool):
         model_config_path = getattr(arguments, 'model_config_path', None)
         if model_config_path is None:
             model_config = {}
         else:
             with open(model_config_path) as f:
                 model_config = json.load(f)
-                if not torch.cuda.is_available():
+                if not torch.cuda.is_available() and pytorch_model:
                     if '__init__' not in model_config:
                         model_config['__init__'] = {}
                     model_config['__init__']['device'] = 'cpu'