diff --git a/.travis.yml b/.travis.yml
index ec39bd97c..60fa30fd3 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -33,14 +33,18 @@ matrix:
     - env: DISTRIB="ubuntu"
     # Latest release
     - env: DISTRIB="conda" PYTHON_VERSION="3.6"
-           NUMPY_VERSION="*" SCIPY_VERSION="*" SKLEARN_VERSION="0.21"
+           NUMPY_VERSION="*" SCIPY_VERSION="*" SKLEARN_VERSION="0.21.2"
+           OPTIONAL_DEPS="true"
     - env: DISTRIB="conda" PYTHON_VERSION="3.7"
-           NUMPY_VERSION="1.14.6" SCIPY_VERSION="1.1.0" SKLEARN_VERSION="0.21"
+           NUMPY_VERSION="*" SCIPY_VERSION="*" SKLEARN_VERSION="0.21.2"
+           OPTIONAL_DEPS="false"
     - env: DISTRIB="conda" PYTHON_VERSION="3.7"
            NUMPY_VERSION="*" SCIPY_VERSION="*" SKLEARN_VERSION="master"
+           OPTIONAL_DEPS="false"
   allow_failures:
     - env: DISTRIB="conda" PYTHON_VERSION="3.7"
            NUMPY_VERSION="*" SCIPY_VERSION="*" SKLEARN_VERSION="master"
+           OPTIONAL_DEPS="false"
 
 install: source build_tools/travis/install.sh
 script: bash build_tools/travis/test_script.sh
diff --git a/build_tools/travis/install.sh b/build_tools/travis/install.sh
index cd31178e2..c5f2534a6 100755
--- a/build_tools/travis/install.sh
+++ b/build_tools/travis/install.sh
@@ -10,10 +10,6 @@
 
 set -e
 
-echo 'List files from cached directories'
-echo 'pip:'
-ls $HOME/.cache/pip
-
 export CC=/usr/lib/ccache/gcc
 export CXX=/usr/lib/ccache/g++
 # Useful for debugging how ccache is used
@@ -32,7 +28,7 @@ if [[ "$DISTRIB" == "conda" ]]; then
     MINICONDA_PATH=/home/travis/miniconda
     chmod +x miniconda.sh && ./miniconda.sh -b -p $MINICONDA_PATH
     export PATH=$MINICONDA_PATH/bin:$PATH
-    conda update --yes conda
+    conda install --yes conda=4.6
 
     # Configure the conda environment and put it in the path using the
     # provided versions
@@ -40,8 +36,7 @@ if [[ "$DISTRIB" == "conda" ]]; then
     source activate testenv
     conda install --yes numpy=$NUMPY_VERSION scipy=$SCIPY_VERSION
 
-    if [[ $PYTHON_VERSION == "3.6" ]]; then
-        # Tensorflow is not available in Python 3.7 yet.
+    if [[ "$OPTIONAL_DEPS" == "true" ]]; then
         conda install --yes pandas keras tensorflow
         KERAS_BACKEND=tensorflow
         python -c "import keras.backend"