diff --git a/.gitignore b/.gitignore index 06522345..91c6c12a 100755 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,7 @@ record/ # Tools tools/dist/ tools/etc/ +opt/ # Third party etc/*.tar.gz diff --git a/Makefile b/Makefile index 31e2cf56..34ba7598 100755 --- a/Makefile +++ b/Makefile @@ -49,8 +49,9 @@ manifest: web-dist: yarn build - rm -f rhasspy_web.tar.gz - tar -czf rhasspy-web-dist.tar.gz dist/ + mkdir -p download + rm -f download/rhasspy-web-dist.tar.gz + tar -czf download/rhasspy-web-dist.tar.gz dist/ # ----------------------------------------------------------------------------- # Documentation diff --git a/bin/rhasspy b/bin/rhasspy index ca95698f..b2d5af93 100755 --- a/bin/rhasspy +++ b/bin/rhasspy @@ -1,14 +1,29 @@ #!/usr/bin/env bash +set -e # Directory of *this* script -DIR="$( cd "$( dirname "$0" )" && pwd )" +this_dir="$( cd "$( dirname "$0" )" && pwd )" -cd "$DIR/.." -source .venv/bin/activate +venv="${this_dir}/../.venv" +if [[ ! -d "${venv}" ]]; then + echo "Missing virtual environment at ${venv}" + echo "Did you run create-venv.sh?" + exit 1 +fi + +source "${venv}/bin/activate" + +# Force .venv/lib to be used +export LD_LIBRARY_PATH="${venv}/lib:${LD_LIBRARY_PATH}" + +# Use local Kaldi +if [[ -d "${this_dir}/../opt/kaldi" ]]; then + export KALDI_PREFIX="${this_dir}/../opt" +fi # Path to sphinxtrain tools if [[ -d "/usr/lib/sphinxtrain" ]]; then - export PATH="/usr/lib/sphinxtrain:$PATH" + export PATH="/usr/lib/sphinxtrain:${PATH}" fi -python3 -m rhasspy "$@" +cd "${this_dir}/.." && python3 -m rhasspy "$@" diff --git a/create-venv.sh b/create-venv.sh index 8a29f28b..a27a7368 100755 --- a/create-venv.sh +++ b/create-venv.sh @@ -1,6 +1,15 @@ #!/usr/bin/env bash set -e +# Process command-line arguments +no_flair="no" +for arg in "$@"; do + shift + case "${arg}" in + "--no-flair") no_flair="yes" ;; + esac +done + # Directory of *this* script DIR="$( cd "$( dirname "$0" )" && pwd )" @@ -10,7 +19,7 @@ mkdir -p "${download_dir}" # CPU architecture CPU_ARCH="$(lscpu | awk '/^Architecture/{print $2}')" -case $CPU_ARCH in +case "${CPU_ARCH}" in x86_64) FRIENDLY_ARCH=amd64 ;; @@ -47,7 +56,7 @@ sudo apt-get install -y python3 python3-pip python3-venv python3-dev \ gfortran libfst-dev \ sphinxbase-utils sphinxtrain pocketsphinx \ jq checkinstall unzip xz-utils \ - libfst-dev libfst-tools curl + curl # ----------------------------------------------------------------------------- # Python 3.6 @@ -73,6 +82,13 @@ if [[ -z "$(which python3.6)" ]]; then sudo make altinstall fi +# ----------------------------------------------------------------------------- +# Download dependencies +# ----------------------------------------------------------------------------- + +echo "Downloading dependencies" +bash download-dependencies.sh "${CPU_ARCH}" + # ----------------------------------------------------------------------------- # Virtual environment # ----------------------------------------------------------------------------- @@ -90,14 +106,28 @@ echo "Creating new virtual environment" mkdir -p "${VENV_PATH}" "${PYTHON}" -m venv "${VENV_PATH}" +# Extract Rhasspy tools +rhasspy_tools_file="${download_dir}/rhasspy-tools_${FRIENDLY_ARCH}.tar.gz" +echo "Extracting tools (${rhasspy_tools_file})" +tar -C "${VENV_PATH}" -xf "${rhasspy_tools_file}" + +# Force .venv/lib to be used +export LD_LIBRARY_PATH="${VENV_PATH}/lib:${LD_LIBRARY_PATH}" + # shellcheck source=/dev/null source "${VENV_PATH}/bin/activate" + +echo "Installing Python requirements" "${PYTHON}" -m pip install wheel -"${PYTHON}" -m pip install -r requirements.txt -# Download dependencies -echo "Downloading dependencies" -bash download-dependencies.sh +requirements_file="${DIR}/requirements.txt" +if [[ ! -z "${no_flair}" ]]; then + echo "Excluding flair from virtual environment" + grep -v flair "${requirements_file}" > "${temp_dir}/requirements.txt" + requirements_file="${temp_dir}/requirements.txt" +fi + +"${PYTHON}" -m pip install -r "${requirements_file}" # ----------------------------------------------------------------------------- # Pocketsphinx for Python @@ -110,15 +140,10 @@ pocketsphinx_file="${download_dir}/pocketsphinx-python.tar.gz" # Snowboy # ----------------------------------------------------------------------------- -case $CPU_ARCH in +case "${CPU_ARCH}" in x86_64|armv7l) snowboy_file="${download_dir}/snowboy-1.3.0.tar.gz" - if [[ ! -f "${snowboy_file}" ]]; then - snowboy_url='https://github.com/Kitt-AI/snowboy/archive/v1.3.0.tar.gz' - echo "Downloading snowboy (${snowboy_url})" - curl -sSfL-o "${snowboy_file}" "${snowboy_url}" - fi - + echo "Installing snowboy" "${PYTHON}" -m pip install "${snowboy_file}" ;; @@ -131,12 +156,13 @@ esac # ----------------------------------------------------------------------------- if [[ -z "$(which precise-engine)" ]]; then - case $CPU_ARCH in + case "${CPU_ARCH}" in x86_64|armv7l) + echo "Installing Mycroft Precise" precise_file="${download_dir}/precise-engine_0.3.0_${CPU_ARCH}.tar.gz" - precise_install='/usr/lib' - sudo tar -C "${precise_install}" -xf "${precise_file}" - sudo ln -s "${precise_install}/precise-engine/precise-engine" '/usr/bin/precise-engine' + precise_install="${VENV_PATH}/lib" + tar -C "${precise_install}" -xf "${precise_file}" + ln -s "${precise_install}/precise-engine/precise-engine" "${VENV_PATH}/bin/precise-engine" ;; *) @@ -145,71 +171,21 @@ if [[ -z "$(which precise-engine)" ]]; then fi # ----------------------------------------------------------------------------- -# Opengrm +# Kaldi # ----------------------------------------------------------------------------- -if [[ -z "$(which ngramcount)" ]]; then - opengrm_file="${download_dir}/opengrm-ngram-1.3.3.tar.gz" - echo "Building Opengrm ${opengrm_file}" - tar -C "${temp_dir}" -xf "${opengrm_file}" && \ - cd "${temp_dir}/opengrm-ngram-1.3.3" && \ - ./configure && \ - make -j 4 && \ - sudo make install && \ - sudo ldconfig -fi - -# ----------------------------------------------------------------------------- -# Phonetisaurus -# ----------------------------------------------------------------------------- - -if [[ -z "$(which phonetisaurus-apply)" ]]; then - case $CPU_ARCH in - x86_64|armv7l|arm64v8) - # Install pre-built package - phonetisaurus_file="${download_dir}/phonetisaurus-2019_${FRIENDLY_ARCH}.deb" - echo "Installing phonetisaurus (${phonetisaurus_file})" - sudo dpkg -i "${phonetisaurus_file}" - ;; - - *) - # Build from source - phonetisaurus_file="${download_dir}/phonetisaurus-2019.zip" - echo "Building phonetisaurus (${phonetisaurus_file})" - unzip -d "${temp_dir}" "${phonetisaurus_file}" && \ - cd "${temp_dir}/phonetisaurus" && \ - ./configure && \ - make -j 4 && \ - sudo make install - esac -fi - -# Add /usr/local/lib to LD_LIBRARY_PATH -sudo ldconfig - -# ----------------------------------------------------------------------------- -# NodeJS / Yarn -# ----------------------------------------------------------------------------- - -if [[ -z "$(which node)" ]]; then - echo "Installing nodejs" - sudo apt-get install -y nodejs -fi - -if [[ -z "$(which yarn)" ]]; then - echo "Installing yarn" - curl -o- -L https://yarnpkg.com/install.sh | bash - - # Need to re-source .bashrc so yarn is in the path - source "${HOME}/.bashrc" -fi +kaldi_file="${download_dir}/kaldi_${FRIENDLY_ARCH}.tar.gz" +echo "Installing Kaldi (${kaldi_file})" +mkdir -p "${DIR}/opt" +tar -C "${DIR}/opt" -xf "${kaldi_file}" # ----------------------------------------------------------------------------- # Web Interface # ----------------------------------------------------------------------------- -echo "Building web interface" -cd "${DIR}" && yarn && yarn build +rhasspy_web_file="${download_dir}/rhasspy-web-dist.tar.gz" +echo "Extracting web interface (${rhasspy_web_file})" +tar -C "${DIR}" -xf "${rhasspy_web_file}" # ----------------------------------------------------------------------------- diff --git a/download-dependencies.sh b/download-dependencies.sh index ebe2d458..66a5c7c8 100755 --- a/download-dependencies.sh +++ b/download-dependencies.sh @@ -8,15 +8,37 @@ DIR="$( cd "$( dirname "$0" )" && pwd )" download_dir="${DIR}/download" mkdir -p "${download_dir}" -# CPU architecture -CPU_ARCHS=("x86_64" "armv7l" "arm64v8") -FRIENDLY_ARCHS=("amd64" "armhf" "aarch64") - declare -A CPU_TO_FRIENDLY CPU_TO_FRIENDLY["x86_64"]="amd64" CPU_TO_FRIENDLY["armv7l"]="armhf" CPU_TO_FRIENDLY["arm64v8"]="aarch64" +# CPU architecture +if [[ -z "$1" ]]; then + CPU_ARCHS=("x86_64" "armv7l" "arm64v8") + FRIENDLY_ARCHS=("amd64" "armhf" "aarch64") +else + CPU_ARCHS=("$1") + FRIENDLY_ARCHS=("${CPU_TO_FRIENDLY[$1]}") +fi + +# ----------------------------------------------------------------------------- +# Rhasspy +# ----------------------------------------------------------------------------- + +for FRIENDLY_ARCH in "${FRIENDLY_ARCHS[@]}"; +do + rhasspy_files=("rhasspy-tools_${FRIENDLY_ARCH}.tar.gz" "rhasspy-web-dist.tar.gz") + for rhasspy_file_name in "${rhasspy_files}"; do + rhasspy_file="${download_dir}/${rhasspy_file_name}" + if [[ ! -f "${rhasspy_file}" ]]; then + rhasspy_file_url="https://github.com/synesthesiam/rhasspy/releases/download/v2.0/${rhasspy_file_name}" + echo "Downloading ${rhasspy_file} (${rhasspy_file_url})" + curl -sSfL -o "${rhasspy_file}" "${rhasspy_file_url}" + fi + done +done + # ----------------------------------------------------------------------------- # Pocketsphinx for Python # ----------------------------------------------------------------------------- @@ -54,57 +76,24 @@ fi # Mycroft Precise # ----------------------------------------------------------------------------- -for CPU_ARCH in "x86_64" "armv7l" -do - precise_file="${download_dir}/precise-engine_0.3.0_${CPU_ARCH}.tar.gz" - if [[ ! -f "${precise_file}" ]]; then - precise_url="https://github.com/MycroftAI/mycroft-precise/releases/download/v0.3.0/precise-engine_0.3.0_${CPU_ARCH}.tar.gz" - echo "Downloading Mycroft Precise (${precise_url})" - curl -sSfL -o "${precise_file}" "${precise_url}" - fi -done - -# ----------------------------------------------------------------------------- -# Opengrm -# ----------------------------------------------------------------------------- - -if [[ -z "$(which ngramcount)" ]]; then - opengrm_file="${download_dir}/opengrm-ngram-1.3.3.tar.gz" - if [[ ! -f "${opengrm_file}" ]]; then - opengrm_url='https://www.opengrm.org/twiki/pub/GRM/NGramDownload/opengrm-ngram-1.3.3.tar.gz' - echo "Download Opengrm (${opengrm_url})" - curl -sSfLk -o "${opengrm_file}" "${opengrm_url}" - fi -fi - -# ----------------------------------------------------------------------------- -# Phonetisaurus -# ----------------------------------------------------------------------------- - -for FRIENDLY_ARCH in "${FRIENDLY_ARCHS[@]}" +for CPU_ARCH in "${CPU_ARCHS}"; do - # Install pre-built package - phonetisaurus_file="${download_dir}/phonetisaurus-2019_${FRIENDLY_ARCH}.deb" - if [[ ! -f "${phonetisaurus_file}" ]]; then - phonetisaurus_url="https://github.com/synesthesiam/phonetisaurus-2019/releases/download/v1.0/phonetisaurus-2019_${FRIENDLY_ARCH}.deb" - echo "Downloading phonetisaurus (${phonetisaurus_url})" - curl -sSfL -o "${phonetisaurus_file}" "${phonetisaurus_url}" - fi + case $CPU_ARCH in + x86_64|armv7l) + precise_file="${download_dir}/precise-engine_0.3.0_${CPU_ARCH}.tar.gz" + if [[ ! -f "${precise_file}" ]]; then + precise_url="https://github.com/MycroftAI/mycroft-precise/releases/download/v0.3.0/precise-engine_0.3.0_${CPU_ARCH}.tar.gz" + echo "Downloading Mycroft Precise (${precise_url})" + curl -sSfL -o "${precise_file}" "${precise_url}" + fi + esac done -# Build from source -phonetisaurus_file="${download_dir}/phonetisaurus-2019.zip" -if [[ ! -f "${phonetisaurus_file}" ]]; then - phonetisaurus_url="https://github.com/synesthesiam/phonetisaurus-2019/releases/download/v1.0/phonetisaurus-2019_${FRIENDLY_ARCH}.deb" - echo "Downloading phonetisaurus source (${phonetisaurus_url})" - curl -sSfL -o "${phonetisaurus_file}" "${phonetisaurus_url}" -fi - # ----------------------------------------------------------------------------- # Kaldi # ----------------------------------------------------------------------------- -for FRIENDLY_ARCH in "${FRIENDLY_ARCHS[@]}" +for FRIENDLY_ARCH in "${FRIENDLY_ARCHS}" do # Install pre-built package kaldi_file="${download_dir}/kaldi_${FRIENDLY_ARCH}.tar.gz" diff --git a/profiles/defaults.json b/profiles/defaults.json index b04f74d7..086089e3 100755 --- a/profiles/defaults.json +++ b/profiles/defaults.json @@ -125,7 +125,7 @@ "custom_words": "custom_words.txt", "dictionary": "dictionary.txt", "graph": "graph", - "kaldi_dir": "/opt/kaldi", + "kaldi_dir": "${KALDI_PREFIX}/kaldi", "language_model": "language_model.txt", "model_dir": "model", "unknown_words": "unknown_words.txt", diff --git a/run-venv.sh b/run-venv.sh index cf99921d..a8748e41 100755 --- a/run-venv.sh +++ b/run-venv.sh @@ -15,5 +15,14 @@ if [[ ! -d "${venv}" ]]; then fi cd "${this_dir}" -source .venv/bin/activate +source "${venv}/bin/activate" + +# Force .venv/lib to be used +export LD_LIBRARY_PATH="${venv}/lib:${LD_LIBRARY_PATH}" + +# Use local Kaldi +if [[ -d "${this_dir}/opt/kaldi" ]]; then + export KALDI_PREFIX="${this_dir}/opt" +fi + python3 app.py "$@"