From 66c4742ff45d154e4d7b7c9bae977625849ee383 Mon Sep 17 00:00:00 2001 From: Fhrozen Date: Wed, 3 Jul 2024 23:19:34 +0900 Subject: [PATCH 1/9] [ci skip] test faiseq install --- tools/installers/install_fairseq.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tools/installers/install_fairseq.sh b/tools/installers/install_fairseq.sh index 61824378f6d..5c12199d5b1 100755 --- a/tools/installers/install_fairseq.sh +++ b/tools/installers/install_fairseq.sh @@ -40,13 +40,12 @@ if "$(pt_plus 1.8.0)" && "${python_36_plus}"; then rm -rf fairseq - # FairSeq Commit id when making this PR: `commit 313ff0581561c7725ea9430321d6af2901573dfb` # git clone --depth 1 https://github.com/pytorch/fairseq.git - # TODO(jiatong): to fix after the issue #4035 is fixed in fairseq + # Using last version. git clone https://github.com/pytorch/fairseq.git - cd fairseq - git checkout -b sync_commit 313ff0581561c7725ea9430321d6af2901573dfb - cd .. + # cd fairseq + # git checkout -b sync_commit 313ff0581561c7725ea9430321d6af2901573dfb + # cd .. python3 -m pip install --editable ./fairseq python3 -m pip install filelock From 3be23b6f8a13442dea4d5f653364919e5ea34a6d Mon Sep 17 00:00:00 2001 From: Fhrozen Date: Mon, 8 Jul 2024 20:18:10 +0900 Subject: [PATCH 2/9] [ci skip] remove fairseq install --- ci/install.sh | 2 +- tools/Makefile | 6 +--- tools/installers/install_fairseq.sh | 55 ----------------------------- 3 files changed, 2 insertions(+), 61 deletions(-) delete mode 100755 tools/installers/install_fairseq.sh diff --git a/ci/install.sh b/ci/install.sh index 711ca6686ee..6ff57e73cba 100755 --- a/ci/install.sh +++ b/ci/install.sh @@ -21,7 +21,7 @@ ${CXX:-g++} -v . ./activate_python.sh # FIXME(kamo): Failed to compile pesq - make TH_VERSION="${TH_VERSION}" WITH_OMP="${WITH_OMP-ON}" all warp-transducer.done chainer_ctc.done nkf.done moses.done mwerSegmenter.done pyopenjtalk.done py3mmseg.done s3prl.done transformers.done phonemizer.done fairseq.done k2.done gtn.done longformer.done whisper.done parallel-wavegan.done muskits.done lora.done sph2pipe + make TH_VERSION="${TH_VERSION}" WITH_OMP="${WITH_OMP-ON}" all warp-transducer.done chainer_ctc.done nkf.done moses.done mwerSegmenter.done pyopenjtalk.done py3mmseg.done s3prl.done transformers.done phonemizer.done k2.done gtn.done longformer.done whisper.done parallel-wavegan.done muskits.done lora.done sph2pipe rm -rf kaldi ) . tools/activate_python.sh diff --git a/tools/Makefile b/tools/Makefile index 8a7a22e8c06..7df9a4c7f1a 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -29,7 +29,7 @@ WITH_OMP=ON all: showenv python conda_packages.done ffmpeg.done sctk check_install python: activate_python.sh packaging.done espnet.done pytorch.done chainer.done fairscale.done torch_optimizer.done -extra: warp-transducer.done chainer_ctc.done nkf.done moses.done mwerSegmenter.done pesq kenlm.done pyopenjtalk.done py3mmseg.done beamformit.done fairseq.done s3prl.done k2.done transformers.done phonemizer.done longformer.done muskits.done whisper.done rvad_fast.done sounfile_test parallel-wavegan.done lora.done sph2pipe +extra: warp-transducer.done chainer_ctc.done nkf.done moses.done mwerSegmenter.done pesq kenlm.done pyopenjtalk.done py3mmseg.done beamformit.done s3prl.done k2.done transformers.done phonemizer.done longformer.done muskits.done whisper.done rvad_fast.done sounfile_test parallel-wavegan.done lora.done sph2pipe activate_python.sh: test -f activate_python.sh || { echo "Error: Run ./setup_python.sh or ./setup_anaconda.sh"; exit 1; } @@ -203,10 +203,6 @@ fairscale.done: espnet.done . ./activate_python.sh && ./installers/install_fairscale.sh touch fairscale.done -fairseq.done: espnet.done - . ./activate_python.sh && ./installers/install_fairseq.sh - touch fairseq.done - s3prl.done: espnet.done . ./activate_python.sh && ./installers/install_s3prl.sh touch s3prl.done diff --git a/tools/installers/install_fairseq.sh b/tools/installers/install_fairseq.sh deleted file mode 100755 index 5c12199d5b1..00000000000 --- a/tools/installers/install_fairseq.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -if [ $# != 0 ]; then - echo "Usage: $0" - exit 1; -fi - -if ! python -c "import packaging.version" &> /dev/null; then - python3 -m pip install packaging -fi -torch_version=$(python3 -c "import torch; print(torch.__version__)") -python_36_plus=$(python3 <= V("3.6"): - print("true") -else: - print("false") -EOF -) - -pt_plus(){ - python3 <= L('$1'): - print("true") -else: - print("false") -EOF -} - -echo "[INFO] torch_version=${torch_version}" - - -if "$(pt_plus 1.8.0)" && "${python_36_plus}"; then - - rm -rf fairseq - - # git clone --depth 1 https://github.com/pytorch/fairseq.git - # Using last version. - git clone https://github.com/pytorch/fairseq.git - # cd fairseq - # git checkout -b sync_commit 313ff0581561c7725ea9430321d6af2901573dfb - # cd .. - python3 -m pip install --editable ./fairseq - python3 -m pip install filelock - -else - echo "[WARNING] fairseq requires pytorch>=1.8.0, python>=3.6" - -fi From 55cb79f475932567e4374930b72e079f95b5cb2b Mon Sep 17 00:00:00 2001 From: Fhrozen Date: Tue, 9 Jul 2024 22:10:12 +0900 Subject: [PATCH 3/9] update install fairseq --- ci/install.sh | 2 +- tools/Makefile | 8 ++++- tools/installers/install_fairseq.sh | 50 +++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 2 deletions(-) create mode 100755 tools/installers/install_fairseq.sh diff --git a/ci/install.sh b/ci/install.sh index 6ff57e73cba..711ca6686ee 100755 --- a/ci/install.sh +++ b/ci/install.sh @@ -21,7 +21,7 @@ ${CXX:-g++} -v . ./activate_python.sh # FIXME(kamo): Failed to compile pesq - make TH_VERSION="${TH_VERSION}" WITH_OMP="${WITH_OMP-ON}" all warp-transducer.done chainer_ctc.done nkf.done moses.done mwerSegmenter.done pyopenjtalk.done py3mmseg.done s3prl.done transformers.done phonemizer.done k2.done gtn.done longformer.done whisper.done parallel-wavegan.done muskits.done lora.done sph2pipe + make TH_VERSION="${TH_VERSION}" WITH_OMP="${WITH_OMP-ON}" all warp-transducer.done chainer_ctc.done nkf.done moses.done mwerSegmenter.done pyopenjtalk.done py3mmseg.done s3prl.done transformers.done phonemizer.done fairseq.done k2.done gtn.done longformer.done whisper.done parallel-wavegan.done muskits.done lora.done sph2pipe rm -rf kaldi ) . tools/activate_python.sh diff --git a/tools/Makefile b/tools/Makefile index 7df9a4c7f1a..0a1ce7272b8 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -29,7 +29,7 @@ WITH_OMP=ON all: showenv python conda_packages.done ffmpeg.done sctk check_install python: activate_python.sh packaging.done espnet.done pytorch.done chainer.done fairscale.done torch_optimizer.done -extra: warp-transducer.done chainer_ctc.done nkf.done moses.done mwerSegmenter.done pesq kenlm.done pyopenjtalk.done py3mmseg.done beamformit.done s3prl.done k2.done transformers.done phonemizer.done longformer.done muskits.done whisper.done rvad_fast.done sounfile_test parallel-wavegan.done lora.done sph2pipe +extra: warp-transducer.done chainer_ctc.done nkf.done moses.done mwerSegmenter.done pesq kenlm.done pyopenjtalk.done py3mmseg.done beamformit.done fairseq.done s3prl.done k2.done transformers.done phonemizer.done longformer.done muskits.done whisper.done rvad_fast.done sounfile_test parallel-wavegan.done lora.done sph2pipe activate_python.sh: test -f activate_python.sh || { echo "Error: Run ./setup_python.sh or ./setup_anaconda.sh"; exit 1; } @@ -203,6 +203,10 @@ fairscale.done: espnet.done . ./activate_python.sh && ./installers/install_fairscale.sh touch fairscale.done +fairseq.done: espnet.done + . ./activate_python.sh && ./installers/install_fairseq.sh + touch fairseq.done + s3prl.done: espnet.done . ./activate_python.sh && ./installers/install_s3prl.sh touch s3prl.done @@ -269,5 +273,7 @@ clean_extra: rm -rf hts_engine_API.done open_jtalk.done pyopenjtalk.done rm -rf muskits.done rm -rf rvad_fast.done + rm -rf chainer espeak-ng festival MBROLA ParallelWaveGAN + rm -rf py3mmseg sctk* speech_tools sph2pipe* ._mwerSegmenter rm -rf nkf mecab swig moses mwerSegmenter rm -rf PESQ PESQ.zip diff --git a/tools/installers/install_fairseq.sh b/tools/installers/install_fairseq.sh new file mode 100755 index 00000000000..1fb01593f36 --- /dev/null +++ b/tools/installers/install_fairseq.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [ $# != 0 ]; then + echo "Usage: $0" + exit 1; +fi + +if ! python -c "import packaging.version" &> /dev/null; then + python3 -m pip install packaging +fi +torch_version=$(python3 -c "import torch; print(torch.__version__)") +python_36_plus=$(python3 <= V("3.6"): + print("true") +else: + print("false") +EOF +) + +pt_plus(){ + python3 <= L('$1'): + print("true") +else: + print("false") +EOF +} + +echo "[INFO] torch_version=${torch_version}" + + +if "$(pt_plus 1.8.0)" && "${python_36_plus}"; then + + rm -rf fairseq + + git clone https://github.com/espnet/fairseq.git + python3 -m pip install --editable ./fairseq + python3 -m pip install filelock + +else + echo "[WARNING] fairseq requires pytorch>=1.8.0, python>=3.6" + +fi From 0a3ffcdb36e5116f6ecb087a271bf43aef96d7f8 Mon Sep 17 00:00:00 2001 From: Fhrozen Date: Thu, 11 Jul 2024 21:23:50 +0900 Subject: [PATCH 4/9] add files to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index d2d9850cfcb..ba5301179ca 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ tmp* __pycache__/ check_autopep8 .coverage +.coverage.* htmlcov coverage.xml* test_utils/bats-core/ From 7e67a1e48f6e3a659842c1045cf13c3435bc4643 Mon Sep 17 00:00:00 2001 From: Fhrozen Date: Thu, 11 Jul 2024 21:50:59 +0900 Subject: [PATCH 5/9] add comment for re-run ci --- tools/installers/install_fairseq.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/installers/install_fairseq.sh b/tools/installers/install_fairseq.sh index 1fb01593f36..9a8329c1b12 100755 --- a/tools/installers/install_fairseq.sh +++ b/tools/installers/install_fairseq.sh @@ -39,7 +39,7 @@ echo "[INFO] torch_version=${torch_version}" if "$(pt_plus 1.8.0)" && "${python_36_plus}"; then rm -rf fairseq - + # NOTE(nelson): using custom fairseq for espnet git clone https://github.com/espnet/fairseq.git python3 -m pip install --editable ./fairseq python3 -m pip install filelock From f68ba82324dda8284fe0a626bbb0ef7dbf3eb636 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 11 Jul 2024 12:52:04 +0000 Subject: [PATCH 6/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tools/installers/install_fairseq.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/installers/install_fairseq.sh b/tools/installers/install_fairseq.sh index 9a8329c1b12..71a613621bf 100755 --- a/tools/installers/install_fairseq.sh +++ b/tools/installers/install_fairseq.sh @@ -39,7 +39,7 @@ echo "[INFO] torch_version=${torch_version}" if "$(pt_plus 1.8.0)" && "${python_36_plus}"; then rm -rf fairseq - # NOTE(nelson): using custom fairseq for espnet + # NOTE(nelson): using custom fairseq for espnet git clone https://github.com/espnet/fairseq.git python3 -m pip install --editable ./fairseq python3 -m pip install filelock From 609d1660597aaa762c0cbdb86d90483594ae36a1 Mon Sep 17 00:00:00 2001 From: Fhrozen Date: Fri, 12 Jul 2024 18:32:47 +0900 Subject: [PATCH 7/9] remove centos7 ci due to eol --- ...{ci_on_ubuntu.yml => ci_on_ubuntu_yml.eol} | 50 ++++++++++--------- README.md | 1 - 2 files changed, 26 insertions(+), 25 deletions(-) rename .github/workflows/{ci_on_ubuntu.yml => ci_on_ubuntu_yml.eol} (92%) diff --git a/.github/workflows/ci_on_ubuntu.yml b/.github/workflows/ci_on_ubuntu_yml.eol similarity index 92% rename from .github/workflows/ci_on_ubuntu.yml rename to .github/workflows/ci_on_ubuntu_yml.eol index de2a05796b9..824a6c974c4 100644 --- a/.github/workflows/ci_on_ubuntu.yml +++ b/.github/workflows/ci_on_ubuntu_yml.eol @@ -106,21 +106,23 @@ jobs: max-parallel: 20 matrix: os: [ubuntu-latest] - python-version: [3.8, 3.9, "3.10"] - pytorch-version: [2.0.1, 2.1.0] + # python-version: [3.8, 3.9, "3.10"] + # pytorch-version: [2.0.1, 2.1.0] + python-version: ["3.10"] + pytorch-version: [2.1.0] chainer-version: [6.0.0] use-conda: [false] - include: - - os: ubuntu-latest - python-version: "3.7" - pytorch-version: 1.12.1 - chainer-verssion: 6.0.0 - use-conda: false - - os: ubuntu-latest - python-version: "3.7" - pytorch-version: 1.13.1 - chainer-verssion: 6.0.0 - use-conda: false + # include: + # - os: ubuntu-latest + # python-version: "3.7" + # pytorch-version: 1.12.1 + # chainer-verssion: 6.0.0 + # use-conda: false + # - os: ubuntu-latest + # python-version: "3.7" + # pytorch-version: 1.13.1 + # chainer-verssion: 6.0.0 + # use-conda: false steps: - uses: actions/checkout@master - uses: actions/cache@v3 @@ -148,9 +150,9 @@ jobs: run: | ./ci/install.sh - - name: test shell - run: | - ./ci/test_shell_espnet2.sh + # - name: test shell + # run: | + # ./ci/test_shell_espnet2.sh - name: test python run: ./ci/test_python_espnet2.sh @@ -195,16 +197,16 @@ jobs: max-parallel: 20 matrix: os: [ubuntu-latest] - python-version: [3.7] - pytorch-version: [1.13.1] + python-version: ["3.10"] + pytorch-version: [2.1.0] chainer-version: [6.0.0] use-conda: [false] - include: - - os: ubuntu-latest - python-version: "3.10" - pytorch-version: 2.1.0 - chainer-verssion: 6.0.0 - use-conda: false + # include: + # - os: ubuntu-latest + # python-version: "3.10" + # pytorch-version: 2.1.0 + # chainer-verssion: 6.0.0 + # use-conda: false steps: - uses: actions/checkout@master - uses: actions/cache@v3 diff --git a/README.md b/README.md index 7634a246d21..c893b8de83e 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,6 @@ |ubuntu/python3.8/pip|||[![ci on ubuntu](https://github.com/espnet/espnet/actions/workflows/ci_on_ubuntu.yml/badge.svg)](https://github.com/espnet/espnet/actions/workflows/ci_on_ubuntu.yml?query=branch%3Amaster)|[![ci on ubuntu](https://github.com/espnet/espnet/actions/workflows/ci_on_ubuntu.yml/badge.svg)](https://github.com/espnet/espnet/actions/workflows/ci_on_ubuntu.yml?query=branch%3Amaster)| |ubuntu/python3.7/pip|[![ci on ubuntu](https://github.com/espnet/espnet/actions/workflows/ci_on_ubuntu.yml/badge.svg)](https://github.com/espnet/espnet/actions/workflows/ci_on_ubuntu.yml?query=branch%3Amaster)|[![ci on ubuntu](https://github.com/espnet/espnet/actions/workflows/ci_on_ubuntu.yml/badge.svg)](https://github.com/espnet/espnet/actions/workflows/ci_on_ubuntu.yml?query=branch%3Amaster)||| |debian11/python3.10/conda||[![ci on debian11](https://github.com/espnet/espnet/actions/workflows/ci_on_debian11.yml/badge.svg)](https://github.com/espnet/espnet/actions/workflows/ci_on_debian11.yml?query=branch%3Amaster)||| -|centos7/python3.10/conda||[![ci on centos7](https://github.com/espnet/espnet/actions/workflows/ci_on_centos7.yml/badge.svg)](https://github.com/espnet/espnet/actions/workflows/ci_on_centos7.yml?query=branch%3Amaster)||| |windows/python3.10/pip||||[![ci on windows](https://github.com/espnet/espnet/actions/workflows/ci_on_windows.yml/badge.svg)](https://github.com/espnet/espnet/actions/workflows/ci_on_windows.yml?query=branch%3Amaster)| |macos/python3.10/pip||||[![ci on macos](https://github.com/espnet/espnet/actions/workflows/ci_on_macos.yml/badge.svg)](https://github.com/espnet/espnet/actions/workflows/ci_on_macos.yml?query=branch%3Amaster)| |macos/python3.10/conda||||[![ci on macos](https://github.com/espnet/espnet/actions/workflows/ci_on_macos.yml/badge.svg)](https://github.com/espnet/espnet/actions/workflows/ci_on_macos.yml?query=branch%3Amaster)| From 28a1ae714489918f279fa998b50cc156c2b7ec12 Mon Sep 17 00:00:00 2001 From: Fhrozen Date: Fri, 12 Jul 2024 18:34:38 +0900 Subject: [PATCH 8/9] fix files extensions --- .github/workflows/{ci_on_centos7.yml => ci_on_centos7_yml.eol} | 0 .github/workflows/{ci_on_ubuntu_yml.eol => ci_on_ubuntu.yml} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ci_on_centos7.yml => ci_on_centos7_yml.eol} (100%) rename .github/workflows/{ci_on_ubuntu_yml.eol => ci_on_ubuntu.yml} (100%) diff --git a/.github/workflows/ci_on_centos7.yml b/.github/workflows/ci_on_centos7_yml.eol similarity index 100% rename from .github/workflows/ci_on_centos7.yml rename to .github/workflows/ci_on_centos7_yml.eol diff --git a/.github/workflows/ci_on_ubuntu_yml.eol b/.github/workflows/ci_on_ubuntu.yml similarity index 100% rename from .github/workflows/ci_on_ubuntu_yml.eol rename to .github/workflows/ci_on_ubuntu.yml From 163b5b3f428c9c55884f1dc5e5a74ed0abc3562a Mon Sep 17 00:00:00 2001 From: Fhrozen Date: Fri, 12 Jul 2024 18:37:12 +0900 Subject: [PATCH 9/9] fix ci ubuntu --- .github/workflows/ci_on_ubuntu.yml | 50 ++++++++++++++---------------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci_on_ubuntu.yml b/.github/workflows/ci_on_ubuntu.yml index 824a6c974c4..de2a05796b9 100644 --- a/.github/workflows/ci_on_ubuntu.yml +++ b/.github/workflows/ci_on_ubuntu.yml @@ -106,23 +106,21 @@ jobs: max-parallel: 20 matrix: os: [ubuntu-latest] - # python-version: [3.8, 3.9, "3.10"] - # pytorch-version: [2.0.1, 2.1.0] - python-version: ["3.10"] - pytorch-version: [2.1.0] + python-version: [3.8, 3.9, "3.10"] + pytorch-version: [2.0.1, 2.1.0] chainer-version: [6.0.0] use-conda: [false] - # include: - # - os: ubuntu-latest - # python-version: "3.7" - # pytorch-version: 1.12.1 - # chainer-verssion: 6.0.0 - # use-conda: false - # - os: ubuntu-latest - # python-version: "3.7" - # pytorch-version: 1.13.1 - # chainer-verssion: 6.0.0 - # use-conda: false + include: + - os: ubuntu-latest + python-version: "3.7" + pytorch-version: 1.12.1 + chainer-verssion: 6.0.0 + use-conda: false + - os: ubuntu-latest + python-version: "3.7" + pytorch-version: 1.13.1 + chainer-verssion: 6.0.0 + use-conda: false steps: - uses: actions/checkout@master - uses: actions/cache@v3 @@ -150,9 +148,9 @@ jobs: run: | ./ci/install.sh - # - name: test shell - # run: | - # ./ci/test_shell_espnet2.sh + - name: test shell + run: | + ./ci/test_shell_espnet2.sh - name: test python run: ./ci/test_python_espnet2.sh @@ -197,16 +195,16 @@ jobs: max-parallel: 20 matrix: os: [ubuntu-latest] - python-version: ["3.10"] - pytorch-version: [2.1.0] + python-version: [3.7] + pytorch-version: [1.13.1] chainer-version: [6.0.0] use-conda: [false] - # include: - # - os: ubuntu-latest - # python-version: "3.10" - # pytorch-version: 2.1.0 - # chainer-verssion: 6.0.0 - # use-conda: false + include: + - os: ubuntu-latest + python-version: "3.10" + pytorch-version: 2.1.0 + chainer-verssion: 6.0.0 + use-conda: false steps: - uses: actions/checkout@master - uses: actions/cache@v3