Using binary distributions for everything

jasonhildebrand · May 22, 2019 · 8d341dc · 8d341dc
1 parent f19782b
commit 8d341dc
Show file tree

Hide file tree

Showing 7 changed files with 123 additions and 132 deletions.
diff --git a/.gitignore b/.gitignore
@@ -44,6 +44,7 @@ record/
 # Tools
 tools/dist/
 tools/etc/
+opt/
 
 # Third party
 etc/*.tar.gz

diff --git a/Makefile b/Makefile
@@ -49,8 +49,9 @@ manifest:
 
 web-dist:
 	yarn build
-	rm -f rhasspy_web.tar.gz
-	tar -czf rhasspy-web-dist.tar.gz dist/
+	mkdir -p download
+	rm -f download/rhasspy-web-dist.tar.gz
+	tar -czf download/rhasspy-web-dist.tar.gz dist/
 
 # -----------------------------------------------------------------------------
 # Documentation

diff --git a/bin/rhasspy b/bin/rhasspy
@@ -1,14 +1,29 @@
 #!/usr/bin/env bash
+set -e
 
 # Directory of *this* script
-DIR="$( cd "$( dirname "$0" )" && pwd )"
+this_dir="$( cd "$( dirname "$0" )" && pwd )"
 
-cd "$DIR/.."
-source .venv/bin/activate
+venv="${this_dir}/../.venv"
+if [[ ! -d "${venv}" ]]; then
+    echo "Missing virtual environment at ${venv}"
+    echo "Did you run create-venv.sh?"
+    exit 1
+fi
+
+source "${venv}/bin/activate"
+
+# Force .venv/lib to be used
+export LD_LIBRARY_PATH="${venv}/lib:${LD_LIBRARY_PATH}"
+
+# Use local Kaldi
+if [[ -d "${this_dir}/../opt/kaldi" ]]; then
+    export KALDI_PREFIX="${this_dir}/../opt"
+fi
 
 # Path to sphinxtrain tools
 if [[ -d "/usr/lib/sphinxtrain" ]]; then
-    export PATH="/usr/lib/sphinxtrain:$PATH"
+    export PATH="/usr/lib/sphinxtrain:${PATH}"
 fi
 
-python3 -m rhasspy "$@"
+cd "${this_dir}/.." && python3 -m rhasspy "$@"
diff --git a/create-venv.sh b/create-venv.sh
@@ -1,6 +1,15 @@
 #!/usr/bin/env bash
 set -e
 
+# Process command-line arguments
+no_flair="no"
+for arg in "$@"; do
+    shift
+    case "${arg}" in
+        "--no-flair") no_flair="yes" ;;
+    esac
+done
+
 # Directory of *this* script
 DIR="$( cd "$( dirname "$0" )" && pwd )"
 
@@ -10,7 +19,7 @@ mkdir -p "${download_dir}"
 
 # CPU architecture
 CPU_ARCH="$(lscpu | awk '/^Architecture/{print $2}')"
-case $CPU_ARCH in
+case "${CPU_ARCH}" in
     x86_64)
         FRIENDLY_ARCH=amd64
         ;;
@@ -47,7 +56,7 @@ sudo apt-get install -y python3 python3-pip python3-venv python3-dev \
      gfortran libfst-dev \
      sphinxbase-utils sphinxtrain pocketsphinx \
      jq checkinstall unzip xz-utils \
-     libfst-dev libfst-tools curl
+     curl
 
 # -----------------------------------------------------------------------------
 # Python 3.6
@@ -73,6 +82,13 @@ if [[ -z "$(which python3.6)" ]]; then
         sudo make altinstall
 fi
 
+# -----------------------------------------------------------------------------
+# Download dependencies
+# -----------------------------------------------------------------------------
+
+echo "Downloading dependencies"
+bash download-dependencies.sh "${CPU_ARCH}"
+
 # -----------------------------------------------------------------------------
 # Virtual environment
 # -----------------------------------------------------------------------------
@@ -90,14 +106,28 @@ echo "Creating new virtual environment"
 mkdir -p "${VENV_PATH}"
 "${PYTHON}" -m venv "${VENV_PATH}"
 
+# Extract Rhasspy tools
+rhasspy_tools_file="${download_dir}/rhasspy-tools_${FRIENDLY_ARCH}.tar.gz"
+echo "Extracting tools (${rhasspy_tools_file})"
+tar -C "${VENV_PATH}" -xf "${rhasspy_tools_file}"
+
+# Force .venv/lib to be used
+export LD_LIBRARY_PATH="${VENV_PATH}/lib:${LD_LIBRARY_PATH}"
+
 # shellcheck source=/dev/null
 source "${VENV_PATH}/bin/activate"
+
+echo "Installing Python requirements"
 "${PYTHON}" -m pip install wheel
-"${PYTHON}" -m pip install -r requirements.txt
 
-# Download dependencies
-echo "Downloading dependencies"
-bash download-dependencies.sh
+requirements_file="${DIR}/requirements.txt"
+if [[ ! -z "${no_flair}" ]]; then
+    echo "Excluding flair from virtual environment"
+    grep -v flair "${requirements_file}" > "${temp_dir}/requirements.txt"
+    requirements_file="${temp_dir}/requirements.txt"
+fi
+
+"${PYTHON}" -m pip install -r "${requirements_file}"
 
 # -----------------------------------------------------------------------------
 # Pocketsphinx for Python
@@ -110,15 +140,10 @@ pocketsphinx_file="${download_dir}/pocketsphinx-python.tar.gz"
 # Snowboy
 # -----------------------------------------------------------------------------
 
-case $CPU_ARCH in
+case "${CPU_ARCH}" in
     x86_64|armv7l)
         snowboy_file="${download_dir}/snowboy-1.3.0.tar.gz"
-        if [[ ! -f "${snowboy_file}" ]]; then
-            snowboy_url='https://github.com/Kitt-AI/snowboy/archive/v1.3.0.tar.gz'
-            echo "Downloading snowboy (${snowboy_url})"
-            curl -sSfL-o "${snowboy_file}" "${snowboy_url}"
-        fi
-
+        echo "Installing snowboy"
         "${PYTHON}" -m pip install "${snowboy_file}"
         ;;
 
@@ -131,12 +156,13 @@ esac
 # -----------------------------------------------------------------------------
 
 if [[ -z "$(which precise-engine)" ]]; then
-    case $CPU_ARCH in
+    case "${CPU_ARCH}" in
         x86_64|armv7l)
+            echo "Installing Mycroft Precise"
             precise_file="${download_dir}/precise-engine_0.3.0_${CPU_ARCH}.tar.gz"
-            precise_install='/usr/lib'
-            sudo tar -C "${precise_install}" -xf "${precise_file}"
-            sudo ln -s "${precise_install}/precise-engine/precise-engine" '/usr/bin/precise-engine'
+            precise_install="${VENV_PATH}/lib"
+            tar -C "${precise_install}" -xf "${precise_file}"
+            ln -s "${precise_install}/precise-engine/precise-engine" "${VENV_PATH}/bin/precise-engine"
             ;;
 
         *)
@@ -145,71 +171,21 @@ if [[ -z "$(which precise-engine)" ]]; then
 fi
 
 # -----------------------------------------------------------------------------
-# Opengrm
+# Kaldi
 # -----------------------------------------------------------------------------
 
-if [[ -z "$(which ngramcount)" ]]; then
-    opengrm_file="${download_dir}/opengrm-ngram-1.3.3.tar.gz"
-    echo "Building Opengrm ${opengrm_file}"
-    tar -C "${temp_dir}" -xf "${opengrm_file}" && \
-        cd "${temp_dir}/opengrm-ngram-1.3.3" && \
-        ./configure && \
-        make -j 4 && \
-        sudo make install && \
-        sudo ldconfig
-fi
-
-# -----------------------------------------------------------------------------
-# Phonetisaurus
-# -----------------------------------------------------------------------------
-
-if [[ -z "$(which phonetisaurus-apply)" ]]; then
-    case $CPU_ARCH in
-        x86_64|armv7l|arm64v8)
-            # Install pre-built package
-            phonetisaurus_file="${download_dir}/phonetisaurus-2019_${FRIENDLY_ARCH}.deb"
-            echo "Installing phonetisaurus (${phonetisaurus_file})"
-            sudo dpkg -i "${phonetisaurus_file}"
-        ;;
-
-        *)
-            # Build from source
-            phonetisaurus_file="${download_dir}/phonetisaurus-2019.zip"
-            echo "Building phonetisaurus (${phonetisaurus_file})"
-            unzip -d "${temp_dir}" "${phonetisaurus_file}" && \
-                cd "${temp_dir}/phonetisaurus" && \
-                ./configure && \
-                make -j 4 && \
-                sudo make install
-    esac
-fi
-
-# Add /usr/local/lib to LD_LIBRARY_PATH
-sudo ldconfig
-
-# -----------------------------------------------------------------------------
-# NodeJS / Yarn
-# -----------------------------------------------------------------------------
-
-if [[ -z "$(which node)" ]]; then
-    echo "Installing nodejs"
-    sudo apt-get install -y nodejs
-fi
-
-if [[ -z "$(which yarn)" ]]; then
-    echo "Installing yarn"
-    curl -o- -L https://yarnpkg.com/install.sh | bash
-
-    # Need to re-source .bashrc so yarn is in the path
-    source "${HOME}/.bashrc"
-fi
+kaldi_file="${download_dir}/kaldi_${FRIENDLY_ARCH}.tar.gz"
+echo "Installing Kaldi (${kaldi_file})"
+mkdir -p "${DIR}/opt"
+tar -C "${DIR}/opt" -xf "${kaldi_file}"
 
 # -----------------------------------------------------------------------------
 # Web Interface
 # -----------------------------------------------------------------------------
 
-echo "Building web interface"
-cd "${DIR}" && yarn && yarn build
+rhasspy_web_file="${download_dir}/rhasspy-web-dist.tar.gz"
+echo "Extracting web interface (${rhasspy_web_file})"
+tar -C "${DIR}" -xf "${rhasspy_web_file}"
 
 # -----------------------------------------------------------------------------
 

diff --git a/download-dependencies.sh b/download-dependencies.sh
@@ -8,15 +8,37 @@ DIR="$( cd "$( dirname "$0" )" && pwd )"
 download_dir="${DIR}/download"
 mkdir -p "${download_dir}"
 
-# CPU architecture
-CPU_ARCHS=("x86_64" "armv7l" "arm64v8")
-FRIENDLY_ARCHS=("amd64" "armhf" "aarch64")
-
 declare -A CPU_TO_FRIENDLY
 CPU_TO_FRIENDLY["x86_64"]="amd64"
 CPU_TO_FRIENDLY["armv7l"]="armhf"
 CPU_TO_FRIENDLY["arm64v8"]="aarch64"
 
+# CPU architecture
+if [[ -z "$1" ]]; then
+    CPU_ARCHS=("x86_64" "armv7l" "arm64v8")
+    FRIENDLY_ARCHS=("amd64" "armhf" "aarch64")
+else
+    CPU_ARCHS=("$1")
+    FRIENDLY_ARCHS=("${CPU_TO_FRIENDLY[$1]}")
+fi
+
+# -----------------------------------------------------------------------------
+# Rhasspy
+# -----------------------------------------------------------------------------
+
+for FRIENDLY_ARCH in "${FRIENDLY_ARCHS[@]}";
+do
+    rhasspy_files=("rhasspy-tools_${FRIENDLY_ARCH}.tar.gz" "rhasspy-web-dist.tar.gz")
+    for rhasspy_file_name in "${rhasspy_files}"; do
+        rhasspy_file="${download_dir}/${rhasspy_file_name}"
+        if [[ ! -f "${rhasspy_file}" ]]; then
+            rhasspy_file_url="https://github.com/synesthesiam/rhasspy/releases/download/v2.0/${rhasspy_file_name}"
+            echo "Downloading ${rhasspy_file} (${rhasspy_file_url})"
+            curl -sSfL -o "${rhasspy_file}" "${rhasspy_file_url}"
+        fi
+    done
+done
+
 # -----------------------------------------------------------------------------
 # Pocketsphinx for Python
 # -----------------------------------------------------------------------------
@@ -54,57 +76,24 @@ fi
 # Mycroft Precise
 # -----------------------------------------------------------------------------
 
-for CPU_ARCH in "x86_64" "armv7l"
-do
-    precise_file="${download_dir}/precise-engine_0.3.0_${CPU_ARCH}.tar.gz"
-    if [[ ! -f "${precise_file}" ]]; then
-        precise_url="https://github.com/MycroftAI/mycroft-precise/releases/download/v0.3.0/precise-engine_0.3.0_${CPU_ARCH}.tar.gz"
-        echo "Downloading Mycroft Precise (${precise_url})"
-        curl -sSfL -o "${precise_file}" "${precise_url}"
-    fi
-done
-
-# -----------------------------------------------------------------------------
-# Opengrm
-# -----------------------------------------------------------------------------
-
-if [[ -z "$(which ngramcount)" ]]; then
-    opengrm_file="${download_dir}/opengrm-ngram-1.3.3.tar.gz"
-    if [[ ! -f "${opengrm_file}" ]]; then
-        opengrm_url='https://www.opengrm.org/twiki/pub/GRM/NGramDownload/opengrm-ngram-1.3.3.tar.gz'
-        echo "Download Opengrm (${opengrm_url})"
-        curl -sSfLk -o "${opengrm_file}" "${opengrm_url}"
-    fi
-fi
-
-# -----------------------------------------------------------------------------
-# Phonetisaurus
-# -----------------------------------------------------------------------------
-
-for FRIENDLY_ARCH in "${FRIENDLY_ARCHS[@]}"
+for CPU_ARCH in "${CPU_ARCHS}";
 do
-    # Install pre-built package
-    phonetisaurus_file="${download_dir}/phonetisaurus-2019_${FRIENDLY_ARCH}.deb"
-    if [[ ! -f "${phonetisaurus_file}" ]]; then
-        phonetisaurus_url="https://github.com/synesthesiam/phonetisaurus-2019/releases/download/v1.0/phonetisaurus-2019_${FRIENDLY_ARCH}.deb"
-        echo "Downloading phonetisaurus (${phonetisaurus_url})"
-        curl -sSfL -o "${phonetisaurus_file}" "${phonetisaurus_url}"
-    fi
+    case $CPU_ARCH in
+        x86_64|armv7l)
+            precise_file="${download_dir}/precise-engine_0.3.0_${CPU_ARCH}.tar.gz"
+            if [[ ! -f "${precise_file}" ]]; then
+                precise_url="https://github.com/MycroftAI/mycroft-precise/releases/download/v0.3.0/precise-engine_0.3.0_${CPU_ARCH}.tar.gz"
+                echo "Downloading Mycroft Precise (${precise_url})"
+                curl -sSfL -o "${precise_file}" "${precise_url}"
+            fi
+    esac
 done
 
-# Build from source
-phonetisaurus_file="${download_dir}/phonetisaurus-2019.zip"
-if [[ ! -f "${phonetisaurus_file}" ]]; then
-    phonetisaurus_url="https://github.com/synesthesiam/phonetisaurus-2019/releases/download/v1.0/phonetisaurus-2019_${FRIENDLY_ARCH}.deb"
-    echo "Downloading phonetisaurus source (${phonetisaurus_url})"
-    curl -sSfL -o "${phonetisaurus_file}" "${phonetisaurus_url}"
-fi
-
 # -----------------------------------------------------------------------------
 # Kaldi
 # -----------------------------------------------------------------------------
 
-for FRIENDLY_ARCH in "${FRIENDLY_ARCHS[@]}"
+for FRIENDLY_ARCH in "${FRIENDLY_ARCHS}"
 do
     # Install pre-built package
     kaldi_file="${download_dir}/kaldi_${FRIENDLY_ARCH}.tar.gz"

diff --git a/profiles/defaults.json b/profiles/defaults.json
@@ -125,7 +125,7 @@
             "custom_words": "custom_words.txt",
             "dictionary": "dictionary.txt",
             "graph": "graph",
-            "kaldi_dir": "/opt/kaldi",
+            "kaldi_dir": "${KALDI_PREFIX}/kaldi",
             "language_model": "language_model.txt",
             "model_dir": "model",
             "unknown_words": "unknown_words.txt",

diff --git a/run-venv.sh b/run-venv.sh
@@ -15,5 +15,14 @@ if [[ ! -d "${venv}" ]]; then
 fi
 
 cd "${this_dir}"
-source .venv/bin/activate
+source "${venv}/bin/activate"
+
+# Force .venv/lib to be used
+export LD_LIBRARY_PATH="${venv}/lib:${LD_LIBRARY_PATH}"
+
+# Use local Kaldi
+if [[ -d "${this_dir}/opt/kaldi" ]]; then
+    export KALDI_PREFIX="${this_dir}/opt"
+fi
+
 python3 app.py "$@"