Connect python virtualenv to tue-env (#688)

Review and merge after #686
tue-robotics · Nov 13, 2023 · 6c1ec71 · 6c1ec71
2 parents 804e1a5 + 0aa67fd
commit 6c1ec71
Show file tree

Hide file tree

Showing 7 changed files with 161 additions and 39 deletions.
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-1.15.2
+1.16.0
diff --git a/dockerfiles/tue-env.Dockerfile b/dockerfiles/tue-env.Dockerfile
@@ -24,6 +24,7 @@ ARG ROS_VERSION=
 ARG ROS_DISTRO=
 # Default is empty and will result in the default targets repo
 ARG TARGETS_REPO=
+ARG CREATE_VENV=false
 ARG OAUTH2_TOKEN=
 
 # Inform scripts that no questions should be asked and set some environment
@@ -78,7 +79,7 @@ RUN --mount=type=ssh,uid=1000 sed -e s/return//g -i ~/.bashrc && \
     export COMMIT=$COMMIT && \
     export REF_NAME=$REF_NAME && \
     # Run the standard installation script
-    source bootstrap.bash --ros-version="${ROS_VERSION}" --ros-distro="${ROS_DISTRO}" --targets-repo="${TARGETS_REPO}" && \
+    source bootstrap.bash --ros-version="${ROS_VERSION}" --ros-distro="${ROS_DISTRO}" --create-virtualenv="${CREATE_VENV}"--targets-repo="${TARGETS_REPO}" && \
     # Make tue-env to be available to the environment
     source ~/.bashrc && \
     # Install target ros

diff --git a/installer/bootstrap.bash b/installer/bootstrap.bash
@@ -4,6 +4,8 @@
 hash git 2> /dev/null || sudo apt-get install --assume-yes -qq git
 # Make sure lsb-release is installed
 hash lsb_release 2> /dev/null || sudo apt-get install --assume-yes -qq lsb-release
+# Make sure python3-virtualenv is installed
+hash virtualenv 2> /dev/null || sudo apt-get install --assume-yes -qq python3-virtualenv
 
 # Check if OS is Ubuntu
 DISTRIB_ID="$(lsb_release -si)"
@@ -31,6 +33,9 @@ do
         --targets-repo=* )
             targets_repo="${i#*=}"
             ;;
+        --create-virtualenv=* )
+            create_virtualenv="${i#*=}"
+            ;;
         * )
             echo "[tue-env](bootstrap) Error! Unknown argument '${i}' provided to bootstrap script."
             exit 1
@@ -106,6 +111,7 @@ esac
 # Script variables
 env_url="https://github.com/tue-robotics/tue-env.git"
 { [[ -n "${targets_repo}" ]] && env_targets_url="${targets_repo}"; } || env_targets_url="https://github.com/tue-robotics/tue-env-targets.git"
+[[ -n "${create_virtualenv}" ]] || create_virtualenv="true"
 env_dir="$HOME/.tue"
 workspace="ros-$TUE_ROS_DISTRO"
 workspace_dir="$HOME/ros/$TUE_ROS_DISTRO"
@@ -166,7 +172,7 @@ source "$env_dir"/setup.bash
 mkdir -p "$workspace_dir"
 
 # Initialize ros environment directory incl. targets
-tue-env init "$workspace" "$workspace_dir" "$env_targets_url"
+tue-env init "${workspace}" "${workspace_dir}" "--create-virtualenv=${create_virtualenv}" "--targets-url=${env_targets_url}"
 
 # Configure environment
 tue-env config "$workspace" set "TUE_ROS_DISTRO" "$TUE_ROS_DISTRO"
@@ -185,6 +191,5 @@ fi
 tue-env set-default "$workspace"
 
 # Activate the default environment
-# No need to follow to file which is already checked by CI
 # shellcheck disable=SC1090
 source "$env_dir"/setup.bash
diff --git a/installer/tue-install-impl.bash b/installer/tue-install-impl.bash
@@ -1104,14 +1104,17 @@ function _tue-install-pip-now
         tue-install-error "Invalid tue-install-pip${pv}-now call: needs package as argument."
     fi
 
+    local user_arg
+    [[ -z "${VIRTUAL_ENV}" ]] && user_arg="--user"
+
     # Make sure pip is up-to-date before checking version and installing
     local pip_version desired_pip_version
     pip_version=$(python"${pv}" -m pip --version | awk '{print $2}')
     desired_pip_version="22"
     if version_gt "$desired_pip_version" "$pip_version"
     then
         tue-install-debug "pip${pv} not yet version >=$desired_pip_version, but $pip_version"
-        tue-install-pipe python"${pv}" -m pip install --user --upgrade pip
+        tue-install-pipe python"${pv}" -m pip install ${user_arg} --upgrade pip
         hash -r
     else
         tue-install-debug "Already pip${pv}>=$desired_pip_version"
@@ -1177,15 +1180,15 @@ function _tue-install-pip-now
     if [ -n "$pips_to_install" ]
     then
         # shellcheck disable=SC2048,SC2086
-        tue-install-pipe python"${pv}" -m pip install --user $pips_to_install <<< yes || tue-install-error "An error occurred while installing pip${pv} packages."
+        tue-install-pipe python"${pv}" -m pip install ${user_arg} $pips_to_install <<< yes || tue-install-error "An error occurred while installing pip${pv} packages."
     fi
 
     if [ -n "$pips_to_install_w_options" ]
     then
         for pkg in $pips_to_install_w_options
         do
             # shellcheck disable=SC2048,SC2086
-            tue-install-pipe python"${pv}" -m pip install --user ${pkg//^/ } <<< yes || tue-install-error "An error occurred while installing pip${pv} packages with options."
+            tue-install-pipe python"${pv}" -m pip install ${user_arg} ${pkg//^/ } <<< yes || tue-install-error "An error occurred while installing pip${pv} packages with options."
         done
     fi
 
@@ -1194,7 +1197,7 @@ function _tue-install-pip-now
         for pkg in $git_pips_to_install
         do
             # shellcheck disable=SC2048,SC2086
-            tue-install-pipe python"${pv}" -m pip install --user $pkg <<< yes || tue-install-error "An error occurred while installing pip${pv} git packages."
+            tue-install-pipe python"${pv}" -m pip install ${user_arg} ${pkg} <<< yes || tue-install-error "An error occurred while installing pip${pv} git packages."
         done
     fi
 }

diff --git a/setup.bash b/setup.bash
@@ -82,6 +82,13 @@ then
     export PATH=$TUE_BIN${PATH:+:${PATH}}
 fi
 
+# Source the python virtual environment if it exists
+if [[ -d "${TUE_ENV_DIR}"/.venv/"${TUE_ENV}" ]]
+then
+    # shellcheck disable=SC1090
+    source "${TUE_ENV_DIR}"/.venv/"${TUE_ENV}"/bin/activate
+fi
+
 if [ -f "$TUE_ENV_DIR"/.env/setup/target_setup.bash ]
 then
     # shellcheck disable=SC1091

diff --git a/setup/tue-env.bash b/setup/tue-env.bash
@@ -16,14 +16,16 @@ function tue-env
     Possible commands:
 
         init           - Initializes new environment
-        remove         - Removes an existing enviroment
+        remove         - Removes an existing environment
         switch         - Switch to a different environment
         config         - Configures current environment
         set-default    - Set default environment
+        unset-default  - Unset default environment
         init-targets   - (Re-)Initialize the target list
         targets        - Changes directory to targets directory
+        init-venv      - Initializes a virtualenv
         list           - List all possible environments
-        list-current   - Shows current environment
+        current        - Shows current environment
         cd             - Changes directory to environment directory
 """
         return 1
@@ -36,22 +38,51 @@ function tue-env
     # Make sure the correct directories are there
     mkdir -p "$TUE_DIR"/user/envs
 
+    local create_venv dir env_name targets_url show_help
+    create_venv="false"
+    show_help="false"
+
     if [[ $cmd == "init" ]]
     then
-        if [ -z "$1" ]
+        if [[ -n "$1" ]]
         then
-            echo "Usage: tue-env init NAME [ DIRECTORY ] [ TARGETS GIT URL ]"
+            env_name=$1
+            shift
+            for i in "$@"
+            do
+                case $i in
+                    --targets-url=* )
+                        targets_url="${i#*=}" ;;
+                    --create-virtualenv=* )
+                        create_venv="${i#*=}" ;;
+                    --help )
+                        show_help="true" ;;
+                    * )
+                        if [[ -z "${dir}" ]]
+                        then
+                            dir="$i"
+                        else
+                            tue-install-error "Unknown input variable $i"
+                        fi
+                        ;;
+                esac
+            done
+        else
+            show_help="true"
+        fi
+
+        if [[ "${show_help}" == "true" ]]
+        then
+            echo "Usage: tue-env init NAME [ DIRECTORY ] [--help] [--targets-url=TARGETS GIT URL] [--create-virtualenv=false|true]"
             return 1
         fi
 
-        local dir
-        dir=$PWD   # default directory is current directory
-        [ -z "$2" ] || dir=$2
-        dir="$( realpath "$dir" )"
+        [[ -z "${dir}" ]] && dir=${PWD} # If no directory is given, use current directory
+        dir="$( realpath "${dir}" )"
 
-        if [ -f "$TUE_DIR"/user/envs/"$1" ]
+        if [ -f "${TUE_DIR}"/user/envs/"${env_name}" ]
         then
-            echo "[tue-env] Environment '$1' already exists"
+            echo "[tue-env] Environment '${env_name}' already exists"
             return 1
         fi
 
@@ -61,15 +92,20 @@ function tue-env
             return 1
         fi
 
-        echo "$dir" > "$TUE_DIR"/user/envs/"$1"
+        echo "${dir}" > "${TUE_DIR}"/user/envs/"${env_name}"
         # Create .env and .env/setup directories
         mkdir -p "$dir"/.env/setup
         echo -e "#! /usr/bin/env bash\n" > "$dir"/.env/setup/user_setup.bash
-        echo "[tue-env] Created new environment $1"
+        echo "[tue-env] Created new environment ${env_name}"
 
-        if [ -n "$3" ]
+        if [[ -n "${targets_url}" ]]
         then
-            tue-env init-targets "$1" "$3"
+            tue-env init-targets "${env_name}" "${targets_url}"
+        fi
+
+        if [[ "${create_venv}" == "true" ]]
+        then
+            tue-env init-venv "${env_name}"
         fi
 
     elif [[ $cmd == "remove" ]]
@@ -87,25 +123,24 @@ options:
             local PURGE env
             PURGE=false
             env=
-            while test $# -gt 0
+            for i in "$@"
             do
-                case "$1" in
+                case $i in
                     --purge)
                         PURGE=true
                         ;;
                     --*)
-                        echo "[tue-env] Unknown option $1"
+                        echo "[tue-env] Unknown option $i"
                         ;;
                     *)
                         # Read only the first passed environment name and ignore
                         # the rest
-                        if [ -z "$env" ]
+                        if [ -z "${env}" ]
                         then
-                            env=$1
+                            env=$i
                         fi
                         ;;
                 esac
-                shift
             done
         fi
 
@@ -173,6 +208,25 @@ Environment directory '${dir}' didn't exist (anymore)"""
         echo "$1" > "$TUE_DIR"/user/config/default_env
         echo "[tue-env] Default environment set to $1"
 
+    elif [[ $cmd == "unset-default" ]]
+    then
+        if [ -n "$1" ]
+        then
+            echo "Usage: tue-env unset-default"
+            echo "No arguments allowed"
+        fi
+
+        if [[ ! -f "${TUE_DIR}"/user/config/default_env ]]
+        then
+            echo "[tue-env] No default environment set, nothing to unset"
+            return 1
+        fi
+        local default_env
+        default_env=$(cat "${TUE_DIR}"/user/config/default_env)
+        rm -f "${TUE_DIR}"/user/config/default_env
+        echo "[tue-env] Default environment '${default_env}' unset"
+        return 0
+
     elif [[ $cmd == "init-targets" ]]
     then
         if [ -z "$1" ] || { [ -z "$TUE_ENV" ] && [ -z "$2" ]; }
@@ -190,7 +244,7 @@ Environment directory '${dir}' didn't exist (anymore)"""
             if [ -z "$env" ]
             then
                 # This shouldn't be possible logical, should have exited after printing usage
-                echo "[tue-env](init-targets) no enviroment set or provided"
+                echo "[tue-env](init-targets) no environment set or provided"
                 return 1
             fi
             url=$1
@@ -225,6 +279,58 @@ Environment directory '${dir}' didn't exist (anymore)"""
             cd "${tue_env_dir}"/.env/targets || { echo -e "Targets directory '${tue_env_dir}/.env/targets' (environment '${env}') does not exist"; return 1; }
         fi
 
+    elif [[ $cmd == "init-venv" ]]
+    then
+        local env
+        env=$1
+        [ -n "${env}" ] || env=${TUE_ENV}
+
+        if [[ -z "${env}" ]]
+        then
+            echo "[tue-env](init-venv) no environment set or provided"
+            echo "Usage: tue-env init-venv [ NAME ]"
+            return 1
+        fi
+
+        local tue_env_dir
+        tue_env_dir=$(cat "${TUE_DIR}"/user/envs/"${env}")
+        local venv_dir
+        venv_dir=${tue_env_dir}/.venv/${env}
+
+        if [ -d "$tue_env_targets_dir" ]
+        then
+            local targets_dir_moved
+            targets_dir_moved=$tue_env_targets_dir.$(date +%F_%R)
+            mv -f "$tue_env_targets_dir" "$targets_dir_moved"
+            echo "[tue-env] Moved old targets of environment '$env' to $targets_dir_moved"
+        fi
+
+        if [[ -d "${venv_dir}" ]]
+        then
+            local venv_dir_moved
+            venv_dir_moved=${venv_dir}.$(date +%F_%R)
+            if [[ $(basename "${VIRTUAL_ENV}") == "${env}" ]]
+            then
+                echo "[tue-env](init-venv) deactivating currently active virtualenv of environment '${env}'"
+                deactivate
+            fi
+            mv -f "${venv_dir}" "${venv_dir_moved}"
+            echo "[tue-env] Moved old virtualenv of environment '${env}' to ${venv_dir_moved}"
+            echo "Don't use it anymore as its old path is hardcoded in the virtualenv"
+        fi
+
+        python3 -m virtualenv "${venv_dir}" -q --system-site-packages --symlinks
+        echo "[tue-env] Initialized virtualenv of environment '${env}'"
+
+        if [ "${env}" == "${TUE_ENV}" ]
+        then
+            local tue_env_dir
+            tue_env_dir=$(cat "${TUE_DIR}"/user/envs/"${env}")
+            # shellcheck disable=SC1090
+            source "${tue_env_dir}"/.venv/"${env}"/bin/activate
+            echo "[tue-env] Activated new virtualenv of currently active environment '${env}'"
+        fi
+
     elif [[ $cmd == "config" ]]
     then
         local env
@@ -254,7 +360,7 @@ Environment directory '${dir}' didn't exist (anymore)"""
             tue_env_dir=$(cat "$TUE_DIR"/user/envs/"$env")
             cd "${tue_env_dir}" || { echo -e "Environment directory '${tue_env_dir}' (environment '${env}') does not exist"; return 1; }
         else
-            echo "[tue-env](cd) no enviroment set or provided"
+            echo "[tue-env](cd) no environment set or provided"
             return 1
         fi
 
@@ -267,13 +373,13 @@ Environment directory '${dir}' didn't exist (anymore)"""
             basename "$env"
         done
 
-    elif [[ $cmd == "list-current" ]]
+    elif [[ $cmd == "current" ]]
     then
         if [[ -n $TUE_ENV ]]
         then
             echo "$TUE_ENV"
         else
-            echo "[tue-env] no enviroment set"
+            echo "[tue-env] no environment set"
         fi
 
     else
@@ -293,11 +399,11 @@ function _tue-env
 
     if [ "$COMP_CWORD" -eq 1 ]
     then
-        mapfile -t COMPREPLY < <(compgen -W "init list switch list-current remove cd set-default config init-targets targets" -- "$cur")
+        mapfile -t COMPREPLY < <(compgen -W "init list switch current remove cd set-default config init-targets targets init-venv" -- "$cur")
     else
         local cmd
         cmd=${COMP_WORDS[1]}
-        if [[ $cmd == "switch" ]] || [[ $cmd == "remove" ]] || [[ $cmd == "cd" ]] || [[ $cmd == "set-default" ]] || [[ $cmd == "init-targets" ]] || [[ $cmd == "targets" ]]
+        if [[ $cmd == "switch" ]] || [[ $cmd == "remove" ]] || [[ $cmd == "cd" ]] || [[ $cmd == "set-default" ]] || [[ $cmd == "init-targets" ]] || [[ $cmd == "targets" ]] || [[ $cmd == "init-venv" ]]
         then
             if [ "$COMP_CWORD" -eq 2 ]
             then