update scikit-learn OneHotEncoder sparse parameter to sparse_output f…

…or newer scikit-learn versions

.github/workflows/CI-python-AutoML.yml

@@ -13,8 +13,8 @@ jobs:
     strategy:
       matrix:
         packageDirectory: ["ml_wrappers"]
-        operatingSystem: [ubuntu-latest, macos-latest, windows-latest]
-        pythonVersion: ['3.7']
+        operatingSystem: [ubuntu-latest]
+        pythonVersion: ['3.8']
 
     runs-on: ${{ matrix.operatingSystem }}
 
@@ -52,7 +52,7 @@ jobs:
     - name: Install pycocotools for automl
       shell: bash -l {0}
       run: |
-        conda install --yes --quiet pycocotools==2.0.4 -c conda-forge
+        conda install --yes --quiet pycocotools==2.0.6 -c conda-forge
     - name: Install dev dependencies
       shell: bash -l {0}
       run: |

.github/workflows/CI-python-minimal.yml

@@ -24,6 +24,11 @@ jobs:
       with:
         auto-update-conda: true
         python-version: ${{ matrix.pythonVersion }}
+    - if: ${{ matrix.operatingSystem == 'macos-latest' }}
+      name: Use Homebrew to install libomp on MacOS
+      shell: bash -l {0}
+      run: |
+        brew install libomp
     - name: Install package
       shell: bash -l {0}
       run: |

.github/workflows/CI-python.yml

@@ -59,6 +59,11 @@ jobs:
       shell: bash -l {0}
       run: |
         conda install --yes -c conda-forge lightgbm
+    - name: Install backwards-compatible keras for transformers
+      shell: bash -l {0}
+      run: |
+        pip install tf-keras
+        pip install keras==2.15
     - name: Install package
       shell: bash -l {0}
       run: |

python/docs/dependencies.rst

@@ -45,7 +45,7 @@ requirements-dev.txt
 - catboost<1.2
 - tensorflow
 - shap
-- transformers<4.20.0
+- transformers<4.40.0
 - datasets
 - raiutils
 - fastai
@@ -59,7 +59,6 @@ requirements-automl.txt
 -----------------------
 
 - mlflow
-- azureml-automl-core
 - azureml-automl-dnn-vision
 - vision_explanation_methods
 

python/ml_wrappers/dataset/dataset_wrapper.py

@@ -9,6 +9,8 @@
 
 import numpy as np
 import pandas as pd
+import sklearn
+from packaging import version
 from scipy.sparse import issparse
 
 from ..common.constants import Defaults
@@ -316,7 +318,11 @@ def one_hot_encode(self, columns):
             from sklearn.preprocessing import OneHotEncoder
         except ImportError:
             return None
-        one_hot_encoder = OneHotEncoder(handle_unknown='ignore', sparse=False)
+        if version.parse(sklearn.__version__) < version.parse('1.2'):
+            ohe_params = {"sparse": False}
+        else:
+            ohe_params = {"sparse_output": False}
+        one_hot_encoder = OneHotEncoder(handle_unknown='ignore', **ohe_params)
         self._one_hot_encoder = ColumnTransformer([('ord', one_hot_encoder, columns)], remainder='passthrough')
         # Note this will change column order, the one hot encoded columns will be at the start and the
         # rest of the columns at the end

python/setup.py

@@ -37,6 +37,7 @@
 
 DEPENDENCIES = [
     'numpy',
+    'packaging',
     'pandas<2.0.0',
     'scipy',
     'scikit-learn'

requirements-automl.txt

@@ -1,4 +1,3 @@
 mlflow
-azureml-automl-core
 azureml-automl-dnn-vision
 vision_explanation_methods

requirements-dev.txt

@@ -3,7 +3,7 @@ xgboost
 catboost<1.2
 tensorflow
 shap
-transformers<4.20.0
+transformers<4.40.0
 datasets
 raiutils
 fastai